@nghyane/arcane 0.1.0

package/src/web/scrapers/mastodon.ts

@@ -0,0 +1,310 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface MastodonAccount {
+	id: string;
+	username: string;
+	acct: string;
+	display_name: string;
+	note: string;
+	url: string;
+	avatar: string;
+	header: string;
+	followers_count: number;
+	following_count: number;
+	statuses_count: number;
+	created_at: string;
+	bot: boolean;
+	fields?: Array<{ name: string; value: string }>;
+}
+
+interface MastodonMediaAttachment {
+	id: string;
+	type: "image" | "video" | "gifv" | "audio" | "unknown";
+	url: string;
+	preview_url?: string;
+	description?: string;
+}
+
+interface MastodonStatus {
+	id: string;
+	created_at: string;
+	content: string;
+	url: string;
+	account: MastodonAccount;
+	reblogs_count: number;
+	favourites_count: number;
+	replies_count: number;
+	reblog?: MastodonStatus;
+	media_attachments: MastodonMediaAttachment[];
+	spoiler_text?: string;
+	sensitive: boolean;
+	visibility: "public" | "unlisted" | "private" | "direct";
+	in_reply_to_id?: string;
+	poll?: {
+		options: Array<{ title: string; votes_count: number }>;
+		votes_count: number;
+		expired: boolean;
+	};
+}
+
+/**
+ * Check if a domain is a Mastodon instance by probing the API
+ */
+async function isMastodonInstance(hostname: string, timeout: number, signal?: AbortSignal): Promise<boolean> {
+	try {
+		const result = await loadPage(`https://${hostname}/api/v1/instance`, {
+			timeout: Math.min(timeout, 5),
+			headers: { Accept: "application/json" },
+			signal,
+		});
+		if (!result.ok) return false;
+		const data = JSON.parse(result.content);
+		// Mastodon instances return uri/domain field
+		return !!(data.uri || data.domain || data.title);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Format a date string to readable format
+ */
+function formatDate(isoDate: string): string {
+	try {
+		const date = new Date(isoDate);
+		return date.toLocaleDateString("en-US", {
+			year: "numeric",
+			month: "short",
+			day: "numeric",
+			hour: "2-digit",
+			minute: "2-digit",
+		});
+	} catch {
+		return isoDate;
+	}
+}
+
+/**
+ * Format a status/post as markdown
+ */
+function formatStatus(status: MastodonStatus, isReblog = false): string {
+	// Handle reblogs (boosts)
+	if (status.reblog && !isReblog) {
+		let md = `🔁 **${status.account.display_name || status.account.username}** boosted:\n\n`;
+		md += formatStatus(status.reblog, true);
+		return md;
+	}
+
+	const account = status.account;
+	let md = "";
+
+	if (!isReblog) {
+		md += `# Post by ${account.display_name || account.username}\n\n`;
+	}
+
+	md += `**@${account.acct}**`;
+	if (account.bot) md += " 🤖";
+	md += ` · ${formatDate(status.created_at)}`;
+	if (status.visibility !== "public") md += ` · ${status.visibility}`;
+	md += "\n\n";
+
+	// Content warning / spoiler
+	if (status.spoiler_text) {
+		md += `> ⚠️ **CW:** ${status.spoiler_text}\n\n`;
+	}
+
+	// Main content (convert HTML to markdown)
+	const content = htmlToBasicMarkdown(status.content);
+	md += `${content}\n\n`;
+
+	// Poll
+	if (status.poll) {
+		md += "**Poll:**\n";
+		for (const option of status.poll.options) {
+			const pct =
+				status.poll.votes_count > 0 ? ((option.votes_count / status.poll.votes_count) * 100).toFixed(1) : "0";
+			md += `- ${option.title} (${pct}%, ${option.votes_count} votes)\n`;
+		}
+		md += `Total: ${status.poll.votes_count} votes${status.poll.expired ? " (closed)" : ""}\n\n`;
+	}
+
+	// Media attachments
+	if (status.media_attachments.length > 0) {
+		md += "**Attachments:**\n";
+		for (const media of status.media_attachments) {
+			const desc = media.description ? ` - ${media.description}` : "";
+			md += `- [${media.type}](${media.url})${desc}\n`;
+		}
+		md += "\n";
+	}
+
+	// Stats
+	md += `---\n`;
+	md += `💬 ${formatCount(status.replies_count)} replies · `;
+	md += `🔁 ${formatCount(status.reblogs_count)} boosts · `;
+	md += `⭐ ${formatCount(status.favourites_count)} favorites\n`;
+
+	return md;
+}
+
+/**
+ * Format an account/profile as markdown
+ */
+function formatAccount(account: MastodonAccount): string {
+	let md = `# ${account.display_name || account.username}\n\n`;
+
+	md += `**@${account.acct}**`;
+	if (account.bot) md += " 🤖 Bot";
+	md += "\n\n";
+
+	// Bio
+	if (account.note) {
+		const bio = htmlToBasicMarkdown(account.note);
+		if (bio && bio !== account.display_name) {
+			md += `${bio}\n\n`;
+		}
+	}
+
+	// Stats
+	md += `**Followers:** ${formatCount(account.followers_count)} · `;
+	md += `**Following:** ${formatCount(account.following_count)} · `;
+	md += `**Posts:** ${formatCount(account.statuses_count)}\n\n`;
+
+	md += `**Joined:** ${formatDate(account.created_at)}\n`;
+	md += `**Profile:** ${account.url}\n`;
+
+	// Profile fields (links, pronouns, etc.)
+	if (account.fields && account.fields.length > 0) {
+		md += "\n**Profile Fields:**\n";
+		for (const field of account.fields) {
+			const value = htmlToBasicMarkdown(field.value);
+			md += `- **${field.name}:** ${value}\n`;
+		}
+	}
+
+	return md;
+}
+
+/**
+ * Handle Mastodon/Fediverse URLs
+ */
+export const handleMastodon: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+
+		// Check for @user/postid or @user pattern
+		const postMatch = parsed.pathname.match(/^\/@([^/]+)\/(\d+)$/);
+		const profileMatch = parsed.pathname.match(/^\/@([^/]+)$/);
+
+		if (!postMatch && !profileMatch) return null;
+
+		// Verify this is a Mastodon instance
+		if (!(await isMastodonInstance(parsed.hostname, timeout, signal))) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+		const instance = parsed.hostname;
+
+		if (postMatch) {
+			// Fetch status/post
+			const [, , statusId] = postMatch;
+			const apiUrl = `https://${instance}/api/v1/statuses/${statusId}`;
+
+			const result = await loadPage(apiUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			if (!result.ok) return null;
+
+			let status: MastodonStatus;
+			try {
+				status = JSON.parse(result.content);
+			} catch {
+				return null;
+			}
+
+			const md = formatStatus(status);
+			const output = finalizeOutput(md);
+
+			return {
+				url,
+				finalUrl: status.url || url,
+				contentType: "text/markdown",
+				method: "mastodon",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: [`Fetched via Mastodon API (${instance})`],
+			};
+		}
+
+		if (profileMatch) {
+			// Fetch account by username lookup
+			const [, username] = profileMatch;
+			const lookupUrl = `https://${instance}/api/v1/accounts/lookup?acct=${encodeURIComponent(username)}`;
+
+			const result = await loadPage(lookupUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			if (!result.ok) return null;
+
+			let account: MastodonAccount;
+			try {
+				account = JSON.parse(result.content);
+			} catch {
+				return null;
+			}
+
+			// Fetch recent statuses
+			const statusesUrl = `https://${instance}/api/v1/accounts/${account.id}/statuses?limit=5&exclude_replies=true`;
+			const statusesResult = await loadPage(statusesUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			let md = formatAccount(account);
+
+			if (statusesResult.ok) {
+				try {
+					const statuses: MastodonStatus[] = JSON.parse(statusesResult.content);
+					if (statuses.length > 0) {
+						md += "\n---\n\n## Recent Posts\n\n";
+						for (const status of statuses.slice(0, 5)) {
+							md += `### ${formatDate(status.created_at)}\n\n`;
+							const content = htmlToBasicMarkdown(status.content);
+							md += `${content}\n\n`;
+							md += `💬 ${status.replies_count} · 🔁 ${status.reblogs_count} · ⭐ ${status.favourites_count}\n\n`;
+						}
+					}
+				} catch {}
+			}
+
+			const output = finalizeOutput(md);
+
+			return {
+				url,
+				finalUrl: account.url || url,
+				contentType: "text/markdown",
+				method: "mastodon",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: [`Fetched via Mastodon API (${instance})`],
+			};
+		}
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/maven.ts

@@ -0,0 +1,152 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface MavenDoc {
+	id: string;
+	g: string; // groupId
+	a: string; // artifactId
+	latestVersion: string;
+	repositoryId: string;
+	p: string; // packaging
+	timestamp: number;
+	versionCount: number;
+	text?: string[];
+	ec?: string[]; // extensions/classifiers
+}
+
+interface MavenResponse {
+	response: {
+		numFound: number;
+		docs: MavenDoc[];
+	};
+}
+
+/**
+ * Handle Maven Central URLs via Solr API
+ * Supports: search.maven.org/artifact/... and mvnrepository.com/artifact/...
+ */
+export const handleMaven: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname;
+
+		// Check if this is a Maven URL
+		const isSearchMaven = hostname === "search.maven.org";
+		const isMvnRepository = hostname === "mvnrepository.com" || hostname === "www.mvnrepository.com";
+
+		if (!isSearchMaven && !isMvnRepository) return null;
+
+		let groupId: string | null = null;
+		let artifactId: string | null = null;
+		let version: string | null = null;
+
+		if (isSearchMaven) {
+			// Pattern: /artifact/{groupId}/{artifactId}[/{version}[/{packaging}]]
+			const match = parsed.pathname.match(/^\/artifact\/([^/]+)\/([^/]+)(?:\/([^/]+))?/);
+			if (!match) return null;
+			groupId = match[1];
+			artifactId = match[2];
+			version = match[3] || null;
+		} else if (isMvnRepository) {
+			// Pattern: /artifact/{groupId}/{artifactId}[/{version}]
+			const match = parsed.pathname.match(/^\/artifact\/([^/]+)\/([^/]+)(?:\/([^/]+))?/);
+			if (!match) return null;
+			groupId = match[1];
+			artifactId = match[2];
+			version = match[3] || null;
+		}
+
+		if (!groupId || !artifactId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Query Maven Central API
+		const apiUrl = `https://search.maven.org/solrsearch/select?q=g:${encodeURIComponent(groupId)}+AND+a:${encodeURIComponent(artifactId)}&wt=json&rows=1`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: MavenResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		if (data.response.numFound === 0) return null;
+
+		const doc = data.response.docs[0];
+		const displayVersion = version || doc.latestVersion;
+
+		let md = `# ${doc.g}:${doc.a}\n\n`;
+		md += `**Group ID:** ${doc.g}\n`;
+		md += `**Artifact ID:** ${doc.a}\n`;
+		md += `**Latest Version:** ${doc.latestVersion}`;
+		if (version && version !== doc.latestVersion) {
+			md += ` (viewing ${version})`;
+		}
+		md += "\n";
+
+		if (doc.p) md += `**Packaging:** ${doc.p}\n`;
+		if (doc.versionCount) md += `**Versions:** ${formatCount(doc.versionCount)}\n`;
+		if (doc.timestamp) {
+			const date = new Date(doc.timestamp);
+			md += `**Last Updated:** ${date.toISOString().split("T")[0]}\n`;
+		}
+
+		// Add dependency snippets
+		md += `\n## Maven Dependency\n\n`;
+		md += "```xml\n";
+		md += `<dependency>\n`;
+		md += `    <groupId>${doc.g}</groupId>\n`;
+		md += `    <artifactId>${doc.a}</artifactId>\n`;
+		md += `    <version>${displayVersion}</version>\n`;
+		md += `</dependency>\n`;
+		md += "```\n";
+
+		md += `\n## Gradle Dependency\n\n`;
+		md += "```groovy\n";
+		md += `implementation '${doc.g}:${doc.a}:${displayVersion}'\n`;
+		md += "```\n";
+
+		md += `\n## Gradle (Kotlin DSL)\n\n`;
+		md += "```kotlin\n";
+		md += `implementation("${doc.g}:${doc.a}:${displayVersion}")\n`;
+		md += "```\n";
+
+		// Add available classifiers/extensions if present
+		if (doc.ec && doc.ec.length > 0) {
+			const extensions = doc.ec.filter(e => e && e !== "-");
+			if (extensions.length > 0) {
+				md += `\n## Available Extensions\n\n`;
+				md += `${extensions.map(e => `- ${e}`).join("\n")}\n`;
+			}
+		}
+
+		md += `\n## Links\n\n`;
+		md += `- [Maven Central](https://search.maven.org/artifact/${doc.g}/${doc.a}/${displayVersion}/jar)\n`;
+		md += `- [MVN Repository](https://mvnrepository.com/artifact/${doc.g}/${doc.a}/${displayVersion})\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "maven",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Maven Central API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/mdn.ts

@@ -0,0 +1,172 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface MDNSection {
+	type: string;
+	value: {
+		id?: string;
+		title?: string;
+		content?: string;
+		isH3?: boolean;
+		code?: string;
+		language?: string;
+		items?: Array<{ term: string; description: string }>;
+		rows?: string[][];
+	};
+}
+
+interface MDNDoc {
+	doc: {
+		title: string;
+		summary: string;
+		mdn_url: string;
+		body: MDNSection[];
+		browserCompat?: unknown;
+	};
+}
+
+/**
+ * Convert MDN body sections to markdown
+ */
+function convertMDNBody(sections: MDNSection[]): string {
+	const parts: string[] = [];
+
+	for (const section of sections) {
+		const { type, value } = section;
+
+		switch (type) {
+			case "prose":
+				if (value.content) {
+					const markdown = htmlToBasicMarkdown(value.content);
+					if (value.title) {
+						const level = value.isH3 ? "###" : "##";
+						parts.push(`${level} ${value.title}\n\n${markdown}`);
+					} else {
+						parts.push(markdown);
+					}
+				}
+				break;
+
+			case "browser_compatibility":
+				if (value.title) {
+					parts.push(`## ${value.title}\n\n(See browser compatibility data at MDN)`);
+				}
+				break;
+
+			case "specifications":
+				if (value.title) {
+					parts.push(`## ${value.title}\n\n(See specifications at MDN)`);
+				}
+				break;
+
+			case "code_example":
+				if (value.title) {
+					parts.push(`### ${value.title}`);
+				}
+				if (value.code) {
+					const lang = value.language || "";
+					parts.push(`\`\`\`${lang}\n${value.code}\n\`\`\``);
+				}
+				break;
+
+			case "definition_list":
+				if (value.items) {
+					for (const item of value.items) {
+						parts.push(`**${item.term}**`);
+						const desc = htmlToBasicMarkdown(item.description);
+						parts.push(desc);
+					}
+				}
+				break;
+
+			case "table":
+				if (value.rows && value.rows.length > 0) {
+					// Simple markdown table
+					const header = value.rows[0].map(cell => htmlToBasicMarkdown(cell)).join(" | ");
+					const separator = value.rows[0].map(() => "---").join(" | ");
+					const bodyRows = value.rows.slice(1).map(row => row.map(cell => htmlToBasicMarkdown(cell)).join(" | "));
+
+					parts.push(`| ${header} |`);
+					parts.push(`| ${separator} |`);
+					for (const row of bodyRows) {
+						parts.push(`| ${row} |`);
+					}
+				}
+				break;
+
+			default:
+				// Skip unknown types
+				break;
+		}
+	}
+
+	return parts.join("\n\n");
+}
+
+export const handleMDN: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	const urlObj = new URL(url);
+
+	// Only handle developer.mozilla.org
+	if (!urlObj.hostname.includes("developer.mozilla.org")) {
+		return null;
+	}
+
+	// Only handle docs paths
+	if (!urlObj.pathname.includes("/docs/")) {
+		return null;
+	}
+
+	const notes: string[] = [];
+
+	// Construct JSON API URL
+	const jsonUrl = url.replace(/\/?$/, "/index.json");
+
+	try {
+		const result = await loadPage(jsonUrl, { timeout, signal, headers: { Accept: "application/json" } });
+
+		if (!result.ok) {
+			notes.push(`Failed to fetch MDN JSON API (status ${result.status || "unknown"})`);
+			return null;
+		}
+
+		const data: MDNDoc = JSON.parse(result.content);
+		const { doc } = data;
+
+		if (!doc || !doc.title) {
+			notes.push("Invalid MDN JSON structure");
+			return null;
+		}
+
+		// Build markdown content
+		const parts: string[] = [];
+
+		parts.push(`# ${doc.title}`);
+
+		if (doc.summary) {
+			const summary = htmlToBasicMarkdown(doc.summary);
+			parts.push(summary);
+		}
+
+		if (doc.body && doc.body.length > 0) {
+			const bodyMarkdown = convertMDNBody(doc.body);
+			parts.push(bodyMarkdown);
+		}
+
+		const rawContent = parts.join("\n\n");
+		const { content, truncated } = finalizeOutput(rawContent);
+
+		return {
+			url,
+			finalUrl: doc.mdn_url || result.finalUrl,
+			contentType: "text/markdown",
+			method: "mdn",
+			content,
+			fetchedAt: new Date().toISOString(),
+			truncated,
+			notes,
+		};
+	} catch (err) {
+		notes.push(`MDN handler error: ${err instanceof Error ? err.message : String(err)}`);
+		return null;
+	}
+};