@nghyane/arcane 0.1.0

package/src/web/scrapers/open-vsx.ts

@@ -0,0 +1,119 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface OpenVsxFileLinks {
+	readme?: string;
+}
+
+interface OpenVsxExtension {
+	name: string;
+	namespace: string;
+	version: string;
+	displayName?: string;
+	description?: string;
+	downloadCount?: number;
+	averageRating?: number;
+	reviewCount?: number;
+	repository?: string | { url?: string };
+	license?: string;
+	categories?: string[];
+	homepage?: string;
+	files?: OpenVsxFileLinks;
+}
+
+/**
+ * Handle Open VSX URLs via their API
+ */
+export const handleOpenVsx: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "open-vsx.org" && parsed.hostname !== "www.open-vsx.org") return null;
+
+		const match = parsed.pathname.match(/^\/extension\/([^/]+)\/([^/]+)(?:\/([^/]+))?\/?$/);
+		if (!match) return null;
+
+		const namespace = decodeURIComponent(match[1]);
+		const extension = decodeURIComponent(match[2]);
+		const version = match[3] ? decodeURIComponent(match[3]) : null;
+
+		const fetchedAt = new Date().toISOString();
+		const baseUrl = `https://open-vsx.org/api/${encodeURIComponent(namespace)}/${encodeURIComponent(extension)}`;
+		const apiUrl = version ? `${baseUrl}/${encodeURIComponent(version)}` : baseUrl;
+
+		const result = await loadPage(apiUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		let data: OpenVsxExtension;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let readme: string | null = null;
+		const readmeUrl = data.files?.readme;
+		if (readmeUrl) {
+			try {
+				const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10), signal });
+				if (readmeResult.ok) readme = readmeResult.content;
+			} catch {}
+		}
+
+		const displayName = data.displayName || data.name || `${namespace}/${extension}`;
+		const displayNamespace = data.namespace || namespace;
+		const displayVersion = data.version || version || "unknown";
+		const downloads = typeof data.downloadCount === "number" ? data.downloadCount : null;
+		const rating = typeof data.averageRating === "number" ? data.averageRating : null;
+		const reviews = typeof data.reviewCount === "number" ? data.reviewCount : null;
+		const repository = typeof data.repository === "string" ? data.repository : data.repository?.url || null;
+
+		let md = `# ${displayName}\n\n`;
+		if (data.description) md += `${data.description}\n\n`;
+
+		md += `**Namespace:** ${displayNamespace}\n`;
+		md += `**Extension:** ${data.name || extension}\n`;
+		md += `**Version:** ${displayVersion}`;
+		if (data.license) md += ` | **License:** ${data.license}`;
+		md += "\n";
+
+		if (downloads !== null) {
+			md += `**Downloads:** ${formatCount(downloads)}\n`;
+		}
+
+		if (rating !== null) {
+			const reviewSuffix = reviews !== null ? ` (${reviews} reviews)` : "";
+			md += `**Rating:** ${rating}${reviewSuffix}\n`;
+		}
+
+		if (repository) {
+			const cleanedRepo = repository.replace(/^git\+/, "").replace(/\.git$/, "");
+			md += `**Repository:** ${cleanedRepo}\n`;
+		}
+
+		if (data.homepage) md += `**Homepage:** ${data.homepage}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+
+		if (readme) {
+			md += "\n---\n\n## README\n\n";
+			md += `${readme}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "open-vsx",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Open VSX API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/opencorporates.ts

@@ -0,0 +1,275 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface Officer {
+	id: number;
+	name: string;
+	position?: string;
+	start_date?: string;
+	end_date?: string;
+	occupation?: string;
+	nationality?: string;
+	inactive?: boolean;
+}
+
+interface Address {
+	street_address?: string;
+	locality?: string;
+	region?: string;
+	postal_code?: string;
+	country?: string;
+}
+
+interface CompanyData {
+	name: string;
+	company_number: string;
+	jurisdiction_code: string;
+	incorporation_date?: string;
+	dissolution_date?: string;
+	company_type?: string;
+	registry_url?: string;
+	branch?: string;
+	branch_status?: string;
+	inactive?: boolean;
+	current_status?: string;
+	created_at?: string;
+	updated_at?: string;
+	retrieved_at?: string;
+	opencorporates_url?: string;
+	source?: {
+		publisher?: string;
+		url?: string;
+		retrieved_at?: string;
+	};
+	registered_address?: Address;
+	registered_address_in_full?: string;
+	industry_codes?: Array<{
+		code: string;
+		description?: string;
+		code_scheme_name?: string;
+	}>;
+	identifiers?: Array<{
+		identifier_system_code: string;
+		identifier_system_name?: string;
+		identifier_uid: string;
+	}>;
+	previous_names?: Array<{
+		company_name: string;
+		con_date?: string;
+	}>;
+	alternative_names?: Array<{
+		company_name: string;
+		type?: string;
+	}>;
+	officers?: Array<{ officer: Officer }>;
+	agent_name?: string;
+	agent_address?: string;
+	number_of_employees?: string;
+	native_company_number?: string;
+}
+
+interface ApiResponse {
+	api_version: string;
+	results: {
+		company: CompanyData;
+	};
+}
+
+/**
+ * Handle OpenCorporates URLs via API
+ */
+export const handleOpenCorporates: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("opencorporates.com")) return null;
+
+		// Extract jurisdiction and company number from /companies/{jurisdiction}/{number}
+		const match = parsed.pathname.match(/^\/companies\/([^/]+)\/([^/]+)/);
+		if (!match) return null;
+
+		const jurisdiction = decodeURIComponent(match[1]);
+		const companyNumber = decodeURIComponent(match[2]);
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from OpenCorporates API
+		const apiUrl = `https://api.opencorporates.com/v0.4/companies/${jurisdiction}/${companyNumber}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: ApiResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const company = data.results?.company;
+		if (!company) return null;
+
+		let md = `# ${company.name}\n\n`;
+
+		// Basic info table
+		md += "| Field | Value |\n|-------|-------|\n";
+		md += `| **Company Number** | ${company.company_number} |\n`;
+		md += `| **Jurisdiction** | ${company.jurisdiction_code.toUpperCase()} |\n`;
+		if (company.current_status) {
+			md += `| **Status** | ${company.current_status} |\n`;
+		}
+		if (company.company_type) {
+			md += `| **Company Type** | ${company.company_type} |\n`;
+		}
+		if (company.incorporation_date) {
+			md += `| **Incorporated** | ${company.incorporation_date} |\n`;
+		}
+		if (company.dissolution_date) {
+			md += `| **Dissolved** | ${company.dissolution_date} |\n`;
+		}
+		if (company.branch) {
+			md += `| **Branch** | ${company.branch}${company.branch_status ? ` (${company.branch_status})` : ""} |\n`;
+		}
+		if (company.native_company_number && company.native_company_number !== company.company_number) {
+			md += `| **Native Number** | ${company.native_company_number} |\n`;
+		}
+		md += "\n";
+
+		// Registered address
+		if (company.registered_address_in_full) {
+			md += `## Registered Address\n\n${company.registered_address_in_full}\n\n`;
+		} else if (company.registered_address) {
+			const addr = company.registered_address;
+			const parts = [addr.street_address, addr.locality, addr.region, addr.postal_code, addr.country].filter(
+				Boolean,
+			);
+			if (parts.length > 0) {
+				md += `## Registered Address\n\n${parts.join(", ")}\n\n`;
+			}
+		}
+
+		// Agent info
+		if (company.agent_name) {
+			md += `## Registered Agent\n\n**${company.agent_name}**`;
+			if (company.agent_address) {
+				md += `\n${company.agent_address}`;
+			}
+			md += "\n\n";
+		}
+
+		// Officers/Directors
+		if (company.officers && company.officers.length > 0) {
+			const activeOfficers = company.officers.filter(o => !o.officer.inactive && !o.officer.end_date);
+			const inactiveOfficers = company.officers.filter(o => o.officer.inactive || o.officer.end_date);
+
+			if (activeOfficers.length > 0) {
+				md += `## Current Officers (${activeOfficers.length})\n\n`;
+				for (const { officer } of activeOfficers) {
+					md += `- **${officer.name}**`;
+					if (officer.position) md += ` - ${officer.position}`;
+					if (officer.start_date) md += ` (since ${officer.start_date})`;
+					if (officer.occupation) md += ` [${officer.occupation}]`;
+					if (officer.nationality) md += ` (${officer.nationality})`;
+					md += "\n";
+				}
+				md += "\n";
+			}
+
+			if (inactiveOfficers.length > 0) {
+				md += `## Former Officers (${inactiveOfficers.length})\n\n`;
+				for (const { officer } of inactiveOfficers.slice(0, 10)) {
+					md += `- **${officer.name}**`;
+					if (officer.position) md += ` - ${officer.position}`;
+					if (officer.start_date && officer.end_date) {
+						md += ` (${officer.start_date} to ${officer.end_date})`;
+					} else if (officer.end_date) {
+						md += ` (until ${officer.end_date})`;
+					}
+					md += "\n";
+				}
+				if (inactiveOfficers.length > 10) {
+					md += `\n*...and ${inactiveOfficers.length - 10} more former officers*\n`;
+				}
+				md += "\n";
+			}
+		}
+
+		// Industry codes
+		if (company.industry_codes && company.industry_codes.length > 0) {
+			md += `## Industry Codes\n\n`;
+			for (const ic of company.industry_codes) {
+				md += `- **${ic.code}**`;
+				if (ic.description) md += `: ${ic.description}`;
+				if (ic.code_scheme_name) md += ` (${ic.code_scheme_name})`;
+				md += "\n";
+			}
+			md += "\n";
+		}
+
+		// Identifiers
+		if (company.identifiers && company.identifiers.length > 0) {
+			md += `## Identifiers\n\n`;
+			for (const id of company.identifiers) {
+				md += `- **${id.identifier_system_name || id.identifier_system_code}**: ${id.identifier_uid}\n`;
+			}
+			md += "\n";
+		}
+
+		// Previous names
+		if (company.previous_names && company.previous_names.length > 0) {
+			md += `## Previous Names\n\n`;
+			for (const pn of company.previous_names) {
+				md += `- ${pn.company_name}`;
+				if (pn.con_date) md += ` (until ${pn.con_date})`;
+				md += "\n";
+			}
+			md += "\n";
+		}
+
+		// Alternative names
+		if (company.alternative_names && company.alternative_names.length > 0) {
+			md += `## Alternative Names\n\n`;
+			for (const an of company.alternative_names) {
+				md += `- ${an.company_name}`;
+				if (an.type) md += ` (${an.type})`;
+				md += "\n";
+			}
+			md += "\n";
+		}
+
+		// Source info
+		md += "---\n\n";
+		if (company.source?.publisher) {
+			md += `**Source:** ${company.source.publisher}`;
+			if (company.source.url) md += ` ([registry](${company.source.url}))`;
+			md += "\n";
+		}
+		if (company.registry_url) {
+			md += `**Official Registry:** ${company.registry_url}\n`;
+		}
+		if (company.retrieved_at) {
+			md += `**Data Retrieved:** ${company.retrieved_at}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: company.opencorporates_url || url,
+			contentType: "text/markdown",
+			method: "opencorporates",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via OpenCorporates API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/openlibrary.ts

@@ -0,0 +1,319 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface OpenLibraryAuthor {
+	name?: string;
+	url?: string;
+}
+
+interface OpenLibrarySubject {
+	name: string;
+	url?: string;
+}
+
+interface OpenLibraryPublisher {
+	name: string;
+}
+
+interface OpenLibraryCover {
+	small?: string;
+	medium?: string;
+	large?: string;
+}
+
+interface OpenLibraryWork {
+	title: string;
+	authors?: Array<{ author: { key: string } }>;
+	description?: string | { value: string };
+	subjects?: string[];
+	subject_places?: string[];
+	subject_times?: string[];
+	covers?: number[];
+	first_publish_date?: string;
+}
+
+interface OpenLibraryEdition {
+	title: string;
+	authors?: Array<{ key: string }>;
+	publishers?: string[];
+	publish_date?: string;
+	number_of_pages?: number;
+	isbn_10?: string[];
+	isbn_13?: string[];
+	covers?: number[];
+	description?: string | { value: string };
+	subjects?: string[];
+	works?: Array<{ key: string }>;
+}
+
+interface OpenLibraryBooksApiResponse {
+	[key: string]: {
+		title: string;
+		authors?: OpenLibraryAuthor[];
+		publishers?: OpenLibraryPublisher[];
+		publish_date?: string;
+		number_of_pages?: number;
+		subjects?: OpenLibrarySubject[];
+		cover?: OpenLibraryCover;
+		url?: string;
+		identifiers?: {
+			isbn_10?: string[];
+			isbn_13?: string[];
+			openlibrary?: string[];
+		};
+	};
+}
+
+/**
+ * Handle Open Library URLs via their API
+ */
+export const handleOpenLibrary: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("openlibrary.org")) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const path = parsed.pathname;
+
+		// Match URL patterns
+		const workMatch = path.match(/^\/works\/(OL\d+W)/i);
+		const editionMatch = path.match(/^\/books\/(OL\d+M)/i);
+		const isbnMatch = path.match(/^\/isbn\/(\d{10}|\d{13})/i);
+
+		let md: string | null = null;
+
+		if (workMatch) {
+			md = await fetchWork(workMatch[1], timeout, signal);
+		} else if (editionMatch) {
+			md = await fetchEdition(editionMatch[1], timeout, signal);
+		} else if (isbnMatch) {
+			md = await fetchByIsbn(isbnMatch[1], timeout, signal);
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "openlibrary",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Open Library API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+async function fetchWork(workId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/works/${workId}.json`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) return null;
+
+	let work: OpenLibraryWork;
+	try {
+		work = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${work.title}\n\n`;
+
+	// Fetch author names if we have author keys
+	if (work.authors?.length) {
+		const authorNames = await fetchAuthorNames(
+			work.authors.map(a => a.author.key),
+			timeout,
+			signal,
+		);
+		if (authorNames.length) {
+			md += `**Authors:** ${authorNames.join(", ")}\n`;
+		}
+	}
+
+	if (work.first_publish_date) {
+		md += `**First Published:** ${work.first_publish_date}\n`;
+	}
+
+	if (work.covers?.length) {
+		const coverId = work.covers[0];
+		md += `**Cover:** https://covers.openlibrary.org/b/id/${coverId}-L.jpg\n`;
+	}
+
+	md += `**Open Library:** https://openlibrary.org/works/${workId}\n`;
+	md += "\n";
+
+	const description = extractDescription(work.description);
+	if (description) {
+		md += `## Description\n\n${description}\n\n`;
+	}
+
+	if (work.subjects?.length) {
+		md += `## Subjects\n\n${work.subjects.slice(0, 20).join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchEdition(editionId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/books/${editionId}.json`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) return null;
+
+	let edition: OpenLibraryEdition;
+	try {
+		edition = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${edition.title}\n\n`;
+
+	// Fetch author names
+	if (edition.authors?.length) {
+		const authorNames = await fetchAuthorNames(
+			edition.authors.map(a => a.key),
+			timeout,
+			signal,
+		);
+		if (authorNames.length) {
+			md += `**Authors:** ${authorNames.join(", ")}\n`;
+		}
+	}
+
+	if (edition.publishers?.length) {
+		md += `**Publishers:** ${edition.publishers.join(", ")}\n`;
+	}
+
+	if (edition.publish_date) {
+		md += `**Published:** ${edition.publish_date}\n`;
+	}
+
+	if (edition.number_of_pages) {
+		md += `**Pages:** ${edition.number_of_pages}\n`;
+	}
+
+	const isbns = [...(edition.isbn_13 || []), ...(edition.isbn_10 || [])];
+	if (isbns.length) {
+		md += `**ISBN:** ${isbns[0]}\n`;
+	}
+
+	if (edition.covers?.length) {
+		const coverId = edition.covers[0];
+		md += `**Cover:** https://covers.openlibrary.org/b/id/${coverId}-L.jpg\n`;
+	}
+
+	md += `**Open Library:** https://openlibrary.org/books/${editionId}\n`;
+
+	if (edition.works?.length) {
+		const workKey = edition.works[0].key.replace("/works/", "");
+		md += `**Work:** https://openlibrary.org/works/${workKey}\n`;
+	}
+
+	md += "\n";
+
+	const description = extractDescription(edition.description);
+	if (description) {
+		md += `## Description\n\n${description}\n\n`;
+	}
+
+	if (edition.subjects?.length) {
+		md += `## Subjects\n\n${edition.subjects.slice(0, 20).join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchByIsbn(isbn: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/api/books?bibkeys=ISBN:${isbn}&format=json&jscmd=data`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) return null;
+
+	let data: OpenLibraryBooksApiResponse;
+	try {
+		data = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	const key = `ISBN:${isbn}`;
+	const book = data[key];
+	if (!book) return null;
+
+	let md = `# ${book.title}\n\n`;
+
+	if (book.authors?.length) {
+		md += `**Authors:** ${book.authors.map(a => a.name).join(", ")}\n`;
+	}
+
+	if (book.publishers?.length) {
+		md += `**Publishers:** ${book.publishers.map(p => p.name).join(", ")}\n`;
+	}
+
+	if (book.publish_date) {
+		md += `**Published:** ${book.publish_date}\n`;
+	}
+
+	if (book.number_of_pages) {
+		md += `**Pages:** ${book.number_of_pages}\n`;
+	}
+
+	md += `**ISBN:** ${isbn}\n`;
+
+	if (book.cover?.large || book.cover?.medium) {
+		md += `**Cover:** ${book.cover.large || book.cover.medium}\n`;
+	}
+
+	if (book.url) {
+		md += `**Open Library:** ${book.url}\n`;
+	}
+
+	md += "\n";
+
+	if (book.subjects?.length) {
+		md += `## Subjects\n\n${book.subjects
+			.slice(0, 20)
+			.map(s => s.name)
+			.join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchAuthorNames(authorKeys: string[], timeout: number, signal?: AbortSignal): Promise<string[]> {
+	const names: string[] = [];
+
+	// Fetch authors in parallel (limit to first 5)
+	const promises = authorKeys.slice(0, 5).map(async key => {
+		const authorKey = key.startsWith("/authors/") ? key : `/authors/${key}`;
+		const apiUrl = `https://openlibrary.org${authorKey}.json`;
+		try {
+			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 5), signal });
+			if (result.ok) {
+				const author = JSON.parse(result.content) as { name?: string };
+				return author.name || null;
+			}
+		} catch {}
+		return null;
+	});
+
+	const results = await Promise.all(promises);
+	for (const name of results) {
+		if (name) names.push(name);
+	}
+
+	return names;
+}
+
+function extractDescription(desc: string | { value: string } | undefined): string | null {
+	if (!desc) return null;
+	if (typeof desc === "string") return desc;
+	return desc.value || null;
+}