@nghyane/arcane 0.1.0

package/src/web/scrapers/stackoverflow.ts

@@ -0,0 +1,124 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface SOQuestion {
+	title: string;
+	body: string;
+	score: number;
+	owner: { display_name: string };
+	creation_date: number;
+	tags: string[];
+	answer_count: number;
+	is_answered: boolean;
+}
+
+interface SOAnswer {
+	body: string;
+	score: number;
+	is_accepted: boolean;
+	owner: { display_name: string };
+	creation_date: number;
+}
+
+// Standalone SE network sites (not *.stackexchange.com subdomains)
+const STANDALONE_SE_SITES: Record<string, string> = {
+	"stackoverflow.com": "stackoverflow",
+	"superuser.com": "superuser",
+	"serverfault.com": "serverfault",
+	"askubuntu.com": "askubuntu",
+	"mathoverflow.net": "mathoverflow",
+	"stackapps.com": "stackapps",
+};
+
+/**
+ * Extract the API site parameter from a Stack Exchange hostname
+ * Returns null if not a recognized SE site
+ */
+function getSiteParam(hostname: string): string | null {
+	// Remove www. prefix if present
+	const host = hostname.replace(/^www\./, "");
+
+	// Check standalone sites first
+	if (STANDALONE_SE_SITES[host]) {
+		return STANDALONE_SE_SITES[host];
+	}
+
+	// Handle *.stackexchange.com subdomains (e.g., unix.stackexchange.com → unix)
+	const seMatch = host.match(/^([a-z0-9-]+)\.stackexchange\.com$/);
+	if (seMatch) {
+		return seMatch[1];
+	}
+
+	return null;
+}
+
+/**
+ * Handle Stack Exchange network URLs via API
+ * Supports stackoverflow.com, *.stackexchange.com, superuser.com, serverfault.com, askubuntu.com, etc.
+ */
+export const handleStackOverflow: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const site = getSiteParam(parsed.hostname);
+		if (!site) return null;
+
+		// Extract question ID from URL patterns like /questions/12345/...
+		const match = parsed.pathname.match(/\/questions\/(\d+)/);
+		if (!match) return null;
+
+		const questionId = match[1];
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch question with answers
+		const apiUrl = `https://api.stackexchange.com/2.3/questions/${questionId}?order=desc&sort=votes&site=${site}&filter=withbody`;
+		const qResult = await loadPage(apiUrl, { timeout, signal });
+
+		if (!qResult.ok) return null;
+
+		const qData = JSON.parse(qResult.content) as { items: SOQuestion[] };
+		if (!qData.items?.length) return null;
+
+		const question = qData.items[0];
+
+		let md = `# ${question.title}\n\n`;
+		md += `**Score:** ${question.score} · **Answers:** ${question.answer_count}`;
+		md += question.is_answered ? " (Answered)" : "";
+		md += `\n**Tags:** ${question.tags.join(", ")}\n`;
+		md += `**Asked by:** ${question.owner.display_name} · ${new Date(question.creation_date * 1000).toISOString().split("T")[0]}\n\n`;
+		md += `---\n\n## Question\n\n${htmlToBasicMarkdown(question.body)}\n\n`;
+
+		// Fetch answers
+		const aUrl = `https://api.stackexchange.com/2.3/questions/${questionId}/answers?order=desc&sort=votes&site=${site}&filter=withbody`;
+		const aResult = await loadPage(aUrl, { timeout, signal });
+
+		if (aResult.ok) {
+			const aData = JSON.parse(aResult.content) as { items: SOAnswer[] };
+			if (aData.items?.length) {
+				md += `---\n\n## Answers\n\n`;
+				for (const answer of aData.items.slice(0, 5)) {
+					const accepted = answer.is_accepted ? " (Accepted)" : "";
+					md += `### Score: ${answer.score}${accepted} · by ${answer.owner.display_name}\n\n`;
+					md += `${htmlToBasicMarkdown(answer.body)}\n\n---\n\n`;
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "stackexchange",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: [`Fetched via Stack Exchange API (site=${site})`],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/terraform.ts

@@ -0,0 +1,304 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface TerraformModule {
+	id: string;
+	namespace: string;
+	name: string;
+	provider: string;
+	version: string;
+	description?: string;
+	source?: string;
+	published_at?: string;
+	downloads: number;
+	verified?: boolean;
+	root?: {
+		inputs?: Array<{
+			name: string;
+			type?: string;
+			description?: string;
+			default?: unknown;
+			required?: boolean;
+		}>;
+		outputs?: Array<{
+			name: string;
+			description?: string;
+		}>;
+		dependencies?: Array<{
+			name: string;
+			source: string;
+			version?: string;
+		}>;
+		resources?: Array<{
+			name: string;
+			type: string;
+		}>;
+	};
+	submodules?: Array<{
+		path: string;
+		name: string;
+	}>;
+}
+
+interface TerraformProvider {
+	id: string;
+	namespace: string;
+	name: string;
+	alias?: string;
+	version: string;
+	description?: string;
+	source?: string;
+	published_at?: string;
+	downloads: number;
+	tier?: string;
+	logo_url?: string;
+	docs?: Array<{
+		id: string;
+		title: string;
+		path: string;
+		slug: string;
+		category: string;
+	}>;
+}
+
+/**
+ * Handle Terraform Registry URLs via API
+ */
+export const handleTerraform: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("registry.terraform.io")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Match module URL: /modules/{namespace}/{name}/{provider}
+		const moduleMatch = parsed.pathname.match(/^\/modules\/([^/]+)\/([^/]+)\/([^/]+)/);
+		if (moduleMatch) {
+			const [, namespace, name, provider] = moduleMatch;
+			return await handleModuleUrl(url, namespace, name, provider, timeout, signal, fetchedAt);
+		}
+
+		// Match provider URL: /providers/{namespace}/{type}
+		const providerMatch = parsed.pathname.match(/^\/providers\/([^/]+)\/([^/]+)/);
+		if (providerMatch) {
+			const [, namespace, type] = providerMatch;
+			return await handleProviderUrl(url, namespace, type, timeout, signal, fetchedAt);
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};
+
+async function handleModuleUrl(
+	url: string,
+	namespace: string,
+	name: string,
+	provider: string,
+	timeout: number,
+	signal: AbortSignal | undefined,
+	fetchedAt: string,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://registry.terraform.io/v1/modules/${namespace}/${name}/${provider}`;
+	const result = await loadPage(apiUrl, {
+		timeout,
+		signal,
+		headers: { Accept: "application/json" },
+	});
+
+	if (!result.ok) return null;
+
+	let mod: TerraformModule;
+	try {
+		mod = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${mod.namespace}/${mod.name}/${mod.provider}\n\n`;
+
+	if (mod.description) md += `${mod.description}\n\n`;
+
+	// Metadata line
+	md += `**Version:** ${mod.version}`;
+	if (mod.verified) md += " ✓ Verified";
+	md += `\n`;
+	md += `**Downloads:** ${formatCount(mod.downloads)}\n`;
+	if (mod.published_at) {
+		md += `**Published:** ${new Date(mod.published_at).toLocaleDateString()}\n`;
+	}
+	if (mod.source) {
+		md += `**Source:** ${mod.source}\n`;
+	}
+	md += "\n";
+
+	// Usage example
+	md += `## Usage\n\n\`\`\`hcl\nmodule "${mod.name}" {\n  source  = "${mod.namespace}/${mod.name}/${mod.provider}"\n  version = "${mod.version}"\n}\n\`\`\`\n\n`;
+
+	// Inputs
+	const inputs = mod.root?.inputs;
+	if (inputs && inputs.length > 0) {
+		md += `## Inputs (${inputs.length})\n\n`;
+		md += "| Name | Type | Required | Description |\n";
+		md += "|------|------|----------|-------------|\n";
+		for (const input of inputs.slice(0, 30)) {
+			const required = (input.required ?? input.default === undefined) ? "Yes" : "No";
+			const type = input.type ?? "any";
+			const desc = (input.description ?? "").replace(/\|/g, "\\|").replace(/\n/g, " ").slice(0, 80);
+			md += `| ${input.name} | \`${type}\` | ${required} | ${desc} |\n`;
+		}
+		if (inputs.length > 30) {
+			md += `\n*... and ${inputs.length - 30} more inputs*\n`;
+		}
+		md += "\n";
+	}
+
+	// Outputs
+	const outputs = mod.root?.outputs;
+	if (outputs && outputs.length > 0) {
+		md += `## Outputs (${outputs.length})\n\n`;
+		for (const output of outputs.slice(0, 20)) {
+			md += `- **${output.name}**`;
+			if (output.description) md += `: ${output.description.replace(/\n/g, " ").slice(0, 100)}`;
+			md += "\n";
+		}
+		if (outputs.length > 20) {
+			md += `\n*... and ${outputs.length - 20} more outputs*\n`;
+		}
+		md += "\n";
+	}
+
+	// Dependencies
+	const deps = mod.root?.dependencies;
+	if (deps && deps.length > 0) {
+		md += `## Dependencies (${deps.length})\n\n`;
+		for (const dep of deps.slice(0, 15)) {
+			md += `- **${dep.name}**: ${dep.source}`;
+			if (dep.version) md += ` (${dep.version})`;
+			md += "\n";
+		}
+		if (deps.length > 15) {
+			md += `\n*... and ${deps.length - 15} more dependencies*\n`;
+		}
+		md += "\n";
+	}
+
+	// Resources
+	const resources = mod.root?.resources;
+	if (resources && resources.length > 0) {
+		md += `## Resources (${resources.length})\n\n`;
+		for (const res of resources.slice(0, 20)) {
+			md += `- \`${res.type}\` (${res.name})\n`;
+		}
+		if (resources.length > 20) {
+			md += `\n*... and ${resources.length - 20} more resources*\n`;
+		}
+		md += "\n";
+	}
+
+	// Submodules
+	if (mod.submodules && mod.submodules.length > 0) {
+		md += `## Submodules (${mod.submodules.length})\n\n`;
+		for (const sub of mod.submodules.slice(0, 10)) {
+			md += `- **${sub.name}**: \`${sub.path}\`\n`;
+		}
+		if (mod.submodules.length > 10) {
+			md += `\n*... and ${mod.submodules.length - 10} more submodules*\n`;
+		}
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "terraform",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via Terraform Registry API"],
+	};
+}
+
+async function handleProviderUrl(
+	url: string,
+	namespace: string,
+	type: string,
+	timeout: number,
+	signal: AbortSignal | undefined,
+	fetchedAt: string,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://registry.terraform.io/v1/providers/${namespace}/${type}`;
+	const result = await loadPage(apiUrl, {
+		timeout,
+		signal,
+		headers: { Accept: "application/json" },
+	});
+
+	if (!result.ok) return null;
+
+	let provider: TerraformProvider;
+	try {
+		provider = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${provider.namespace}/${provider.name}\n\n`;
+
+	if (provider.description) md += `${provider.description}\n\n`;
+
+	// Metadata
+	md += `**Version:** ${provider.version}\n`;
+	if (provider.tier) md += `**Tier:** ${provider.tier}\n`;
+	md += `**Downloads:** ${formatCount(provider.downloads)}\n`;
+	if (provider.published_at) {
+		md += `**Published:** ${new Date(provider.published_at).toLocaleDateString()}\n`;
+	}
+	if (provider.source) {
+		md += `**Source:** ${provider.source}\n`;
+	}
+	md += "\n";
+
+	// Usage example
+	md += `## Usage\n\n\`\`\`hcl\nterraform {\n  required_providers {\n    ${provider.name} = {\n      source  = "${provider.namespace}/${provider.name}"\n      version = "~> ${provider.version}"\n    }\n  }\n}\n\nprovider "${provider.name}" {\n  # Configuration options\n}\n\`\`\`\n\n`;
+
+	// Documentation summary
+	if (provider.docs && provider.docs.length > 0) {
+		const categories = new Map<string, typeof provider.docs>();
+		for (const doc of provider.docs) {
+			const cat = doc.category || "other";
+			if (!categories.has(cat)) categories.set(cat, []);
+			categories.get(cat)!.push(doc);
+		}
+
+		md += `## Documentation\n\n`;
+		for (const [category, docs] of categories) {
+			md += `### ${category.charAt(0).toUpperCase() + category.slice(1)} (${docs.length})\n\n`;
+			for (const doc of docs.slice(0, 15)) {
+				md += `- [${doc.title}](https://registry.terraform.io/providers/${namespace}/${type}/latest/docs/${doc.category}/${doc.slug})\n`;
+			}
+			if (docs.length > 15) {
+				md += `\n*... and ${docs.length - 15} more*\n`;
+			}
+			md += "\n";
+		}
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "terraform",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via Terraform Registry API"],
+	};
+}

package/src/web/scrapers/tldr.ts

@@ -0,0 +1,51 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const TLDR_BASE = "https://raw.githubusercontent.com/tldr-pages/tldr/main/pages";
+const PLATFORMS = ["common", "linux", "osx"] as const;
+
+/**
+ * Handle tldr page URLs
+ * - https://tldr.sh/{command}
+ * - https://tldr.ostera.io/{command}
+ */
+export const handleTldr: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "tldr.sh" && parsed.hostname !== "tldr.ostera.io") return null;
+
+		// Extract command from path (e.g., /tar -> tar)
+		const command = parsed.pathname.replace(/^\//, "").replace(/\.md$/, "");
+		if (!command || command.includes("/")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try platforms in order: common, linux, osx
+		for (const platform of PLATFORMS) {
+			const rawUrl = `${TLDR_BASE}/${platform}/${command}.md`;
+			const result = await loadPage(rawUrl, { timeout, signal });
+
+			if (result.ok && result.content.trim()) {
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: rawUrl,
+					contentType: "text/markdown",
+					method: "tldr",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes: [`Fetched from tldr-pages (${platform})`],
+				};
+			}
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/twitter.ts

@@ -0,0 +1,97 @@
+import { parse as parseHtml } from "node-html-parser";
+import { ToolAbortError } from "../../tools/tool-errors";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const NITTER_INSTANCES = [
+	"nitter.privacyredirect.com",
+	"nitter.tiekoetter.com",
+	"nitter.poast.org",
+	"nitter.woodland.cafe",
+];
+
+/**
+ * Handle Twitter/X URLs via Nitter
+ */
+export const handleTwitter: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!["twitter.com", "x.com", "www.twitter.com", "www.x.com"].includes(parsed.hostname)) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try Nitter instances
+		for (const instance of NITTER_INSTANCES) {
+			const nitterUrl = `https://${instance}${parsed.pathname}`;
+			const result = await loadPage(nitterUrl, { timeout: Math.min(timeout, 10), signal });
+
+			if (result.ok && result.content.length > 500) {
+				// Parse the Nitter HTML
+				const doc = parseHtml(result.content);
+
+				// Extract tweet content
+				const tweetContent = doc.querySelector(".tweet-content")?.text?.trim();
+				const fullname = doc.querySelector(".fullname")?.text?.trim();
+				const username = doc.querySelector(".username")?.text?.trim();
+				const date = doc.querySelector(".tweet-date a")?.text?.trim();
+				const stats = doc.querySelector(".tweet-stats")?.text?.trim();
+
+				if (tweetContent) {
+					let md = `# Tweet by ${fullname || "Unknown"} (${username || "@?"})\n\n`;
+					if (date) md += `*${date}*\n\n`;
+					md += `${tweetContent}\n\n`;
+					if (stats) md += `---\n${stats.replace(/\s+/g, " ")}\n`;
+
+					// Check for replies/thread
+					const replies = doc.querySelectorAll(".timeline-item .tweet-content");
+					if (replies.length > 1) {
+						md += `\n---\n\n## Thread/Replies\n\n`;
+						for (const reply of Array.from(replies).slice(1, 10)) {
+							const replyUser = reply.parentNode?.querySelector(".username")?.text?.trim();
+							md += `**${replyUser || "@?"}**: ${reply.text?.trim()}\n\n`;
+						}
+					}
+
+					const output = finalizeOutput(md);
+					return {
+						url,
+						finalUrl: nitterUrl,
+						contentType: "text/markdown",
+						method: "twitter-nitter",
+						content: output.content,
+						fetchedAt,
+						truncated: output.truncated,
+						notes: [`Via Nitter: ${instance}`],
+					};
+				}
+			}
+		}
+	} catch {
+		if (signal?.aborted) {
+			throw new ToolAbortError();
+		}
+	}
+
+	if (signal?.aborted) {
+		throw new ToolAbortError();
+	}
+
+	// X.com blocks all bots - return a helpful error instead of falling through
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/plain",
+		method: "twitter-blocked",
+		content:
+			"Twitter/X blocks automated access. Nitter instances were unavailable.\n\nTry:\n- Opening the link in a browser\n- Using a different Nitter instance manually\n- Checking if the tweet is available via an archive service",
+		fetchedAt: new Date().toISOString(),
+		truncated: false,
+		notes: ["X.com blocks bots; Nitter instances unavailable"],
+	};
+};