@nghyane/arcane 0.1.0

package/src/web/scrapers/w3c.ts

@@ -0,0 +1,163 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+type JsonRecord = Record<string, unknown>;
+
+function asRecord(value: unknown): JsonRecord | null {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+	return value as JsonRecord;
+}
+
+function getString(record: JsonRecord | null, key: string): string | undefined {
+	if (!record) return undefined;
+	const value = record[key];
+	return typeof value === "string" ? value : undefined;
+}
+
+function getRecord(record: JsonRecord | null, key: string): JsonRecord | null {
+	if (!record) return null;
+	return asRecord(record[key]);
+}
+
+function getArray(record: JsonRecord | null, key: string): unknown[] | undefined {
+	if (!record) return undefined;
+	const value = record[key];
+	return Array.isArray(value) ? value : undefined;
+}
+
+function extractShortname(pathname: string): string | null {
+	const trimmed = pathname.replace(/\/+$/g, "");
+	const segments = trimmed.split("/").filter(Boolean);
+
+	if (segments.length < 2 || segments[0] !== "TR") return null;
+
+	if (segments.length === 2) {
+		const shortname = segments[1];
+		if (/^\d{4}$/.test(shortname)) return null;
+		return decodeURIComponent(shortname);
+	}
+
+	if (segments.length >= 3 && /^\d{4}$/.test(segments[1])) {
+		const version = segments[2];
+		const match = version.match(/^[A-Za-z]+-(.+)-\d{8}$/);
+		if (match?.[1]) return decodeURIComponent(match[1]);
+	}
+
+	return null;
+}
+
+function normalizeStatus(status?: string): { code?: string; label?: string } {
+	if (!status) return {};
+	const lower = status.toLowerCase();
+
+	if (lower.includes("working draft")) return { code: "WD", label: status };
+	if (lower.includes("candidate recommendation")) return { code: "CR", label: status };
+	if (lower.includes("proposed recommendation")) return { code: "PR", label: status };
+	if (lower.includes("recommendation")) return { code: "REC", label: status };
+
+	return { label: status };
+}
+
+function extractEditors(editorsPayload: JsonRecord | null): string[] {
+	const links = getRecord(editorsPayload, "_links");
+	const editors = getArray(links, "editors") ?? [];
+	const names: string[] = [];
+
+	for (const entry of editors) {
+		const record = asRecord(entry);
+		const title = getString(record, "title");
+		if (title) names.push(title);
+	}
+
+	return names;
+}
+
+export const handleW3c: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "www.w3.org" && parsed.hostname !== "w3.org") return null;
+
+		const shortname = extractShortname(parsed.pathname);
+		if (!shortname) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		const specUrl = `https://api.w3.org/specifications/${encodeURIComponent(shortname)}`;
+		const latestUrl = `https://api.w3.org/specifications/${encodeURIComponent(shortname)}/versions/latest`;
+
+		const [specResult, latestResult] = await Promise.all([
+			loadPage(specUrl, { timeout, signal, headers: { Accept: "application/json" } }),
+			loadPage(latestUrl, { timeout, signal, headers: { Accept: "application/json" } }),
+		]);
+
+		if (!specResult.ok || !latestResult.ok) return null;
+
+		const specPayload = asRecord(JSON.parse(specResult.content));
+		const latestPayload = asRecord(JSON.parse(latestResult.content));
+		if (!specPayload || !latestPayload) return null;
+
+		const title = getString(specPayload, "title");
+		const shortnameValue = getString(specPayload, "shortname") ?? shortname;
+		const description = getString(specPayload, "description") ?? getString(specPayload, "abstract");
+		const abstract = description ? htmlToBasicMarkdown(description) : undefined;
+
+		const latestVersionUrl =
+			getString(latestPayload, "uri") ??
+			getString(latestPayload, "shortlink") ??
+			getString(specPayload, "shortlink");
+
+		const latestStatus = getString(latestPayload, "status");
+		const normalizedStatus = normalizeStatus(latestStatus);
+
+		const specLinks = getRecord(specPayload, "_links");
+		const historyUrl = getString(getRecord(specLinks, "version-history"), "href");
+
+		const latestLinks = getRecord(latestPayload, "_links");
+		const editorsUrl = getString(getRecord(latestLinks, "editors"), "href");
+
+		let editors: string[] = [];
+		if (editorsUrl) {
+			const editorsResult = await loadPage(editorsUrl, { timeout: Math.min(timeout, 10), signal });
+			if (editorsResult.ok) {
+				try {
+					const editorsPayload = asRecord(JSON.parse(editorsResult.content));
+					editors = editorsPayload ? extractEditors(editorsPayload) : [];
+				} catch {}
+			}
+		}
+
+		let md = `# ${title ?? shortnameValue}\n\n`;
+		if (abstract) md += `## Abstract\n\n${abstract}\n\n`;
+
+		md += "## Metadata\n\n";
+		md += `**Shortname:** ${shortnameValue}\n`;
+		if (normalizedStatus.code) {
+			md += `**Status:** ${normalizedStatus.code}`;
+			if (normalizedStatus.label) md += ` (${normalizedStatus.label})`;
+			md += "\n";
+		} else if (normalizedStatus.label) {
+			md += `**Status:** ${normalizedStatus.label}\n`;
+		}
+		if (editors.length) md += `**Editors:** ${editors.join(", ")}\n`;
+		if (latestVersionUrl) md += `**Latest Version:** ${latestVersionUrl}\n`;
+		if (historyUrl) md += `**History:** ${historyUrl}\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: latestVersionUrl ?? url,
+			contentType: "text/markdown",
+			method: "w3c-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via W3C API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/wikidata.ts

@@ -0,0 +1,357 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Common Wikidata property IDs mapped to human-readable names
+ */
+const PROPERTY_LABELS: Record<string, string> = {
+	P31: "Instance of",
+	P279: "Subclass of",
+	P17: "Country",
+	P131: "Located in",
+	P625: "Coordinates",
+	P18: "Image",
+	P154: "Logo",
+	P571: "Founded",
+	P576: "Dissolved",
+	P169: "CEO",
+	P112: "Founded by",
+	P159: "Headquarters",
+	P452: "Industry",
+	P1128: "Employees",
+	P2139: "Revenue",
+	P856: "Website",
+	P21: "Sex/Gender",
+	P27: "Citizenship",
+	P569: "Born",
+	P570: "Died",
+	P19: "Birthplace",
+	P20: "Death place",
+	P106: "Occupation",
+	P108: "Employer",
+	P69: "Educated at",
+	P22: "Father",
+	P25: "Mother",
+	P26: "Spouse",
+	P40: "Child",
+	P166: "Award",
+	P136: "Genre",
+	P495: "Country of origin",
+	P577: "Publication date",
+	P50: "Author",
+	P123: "Publisher",
+	P364: "Original language",
+	P86: "Composer",
+	P57: "Director",
+	P161: "Cast member",
+	P170: "Creator",
+	P178: "Developer",
+	P275: "License",
+	P306: "Operating system",
+	P277: "Programming language",
+	P348: "Version",
+	P1566: "GeoNames ID",
+	P214: "VIAF ID",
+	P227: "GND ID",
+	P213: "ISNI",
+	P496: "ORCID",
+};
+
+interface WikidataEntity {
+	type: string;
+	id: string;
+	labels?: Record<string, { language: string; value: string }>;
+	descriptions?: Record<string, { language: string; value: string }>;
+	aliases?: Record<string, Array<{ language: string; value: string }>>;
+	claims?: Record<string, WikidataClaim[]>;
+	sitelinks?: Record<string, { site: string; title: string }>;
+}
+
+interface WikidataClaim {
+	mainsnak: {
+		snaktype: string;
+		property: string;
+		datavalue?: {
+			type: string;
+			value: WikidataValue;
+		};
+	};
+	rank: string;
+}
+
+type WikidataValue =
+	| string
+	| { "entity-type": string; id: string; "numeric-id": number }
+	| { time: string; precision: number; calendarmodel: string }
+	| { amount: string; unit: string }
+	| { text: string; language: string }
+	| { latitude: number; longitude: number; precision: number };
+
+/**
+ * Handle Wikidata URLs via EntityData API
+ */
+export const handleWikidata: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("wikidata.org")) return null;
+
+		// Extract Q-id from /wiki/Q123 or /entity/Q123
+		const qidMatch = parsed.pathname.match(/\/(?:wiki|entity)\/(Q\d+)/i);
+		if (!qidMatch) return null;
+
+		const qid = qidMatch[1].toUpperCase();
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch entity data from API
+		const apiUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		let data: { entities: Record<string, WikidataEntity> };
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const entity = data.entities[qid];
+		if (!entity) return null;
+
+		// Get label and description (prefer English)
+		const label = getLocalizedValue(entity.labels, "en") || qid;
+		const description = getLocalizedValue(entity.descriptions, "en");
+		const aliases = getLocalizedAliases(entity.aliases, "en");
+
+		let md = `# ${label} (${qid})\n\n`;
+		if (description) md += `*${description}*\n\n`;
+		if (aliases.length > 0) md += `**Also known as:** ${aliases.join(", ")}\n\n`;
+
+		// Count sitelinks
+		const sitelinkCount = entity.sitelinks ? Object.keys(entity.sitelinks).length : 0;
+		if (sitelinkCount > 0) {
+			md += `**Wikipedia articles:** ${formatCount(sitelinkCount)} languages\n\n`;
+		}
+
+		// Process claims
+		if (entity.claims && Object.keys(entity.claims).length > 0) {
+			md += "## Properties\n\n";
+
+			// Collect entity IDs we need to resolve
+			const entityIdsToResolve = new Set<string>();
+			for (const claims of Object.values(entity.claims)) {
+				for (const claim of claims) {
+					if (claim.mainsnak.datavalue?.type === "wikibase-entityid") {
+						const val = claim.mainsnak.datavalue.value as { id: string };
+						entityIdsToResolve.add(val.id);
+					}
+				}
+			}
+
+			// Fetch labels for referenced entities (limit to 50)
+			const entityLabels = await resolveEntityLabels(Array.from(entityIdsToResolve).slice(0, 50), timeout, signal);
+
+			// Group claims by property
+			const processedProperties: string[] = [];
+			for (const [propId, claims] of Object.entries(entity.claims)) {
+				const propLabel = PROPERTY_LABELS[propId] || propId;
+				const values: string[] = [];
+
+				for (const claim of claims) {
+					if (claim.rank === "deprecated") continue;
+					const value = formatClaimValue(claim, entityLabels);
+					if (value && !values.includes(value)) {
+						values.push(value);
+					}
+				}
+
+				if (values.length > 0) {
+					// Limit values shown per property
+					const displayValues = values.slice(0, 10);
+					const overflow = values.length > 10 ? ` (+${values.length - 10} more)` : "";
+					processedProperties.push(`- **${propLabel}:** ${displayValues.join(", ")}${overflow}`);
+				}
+			}
+
+			// Sort: known properties first, then by property ID
+			processedProperties.sort((a, b) => {
+				const aKnown = Object.values(PROPERTY_LABELS).some(l => a.includes(`**${l}:**`));
+				const bKnown = Object.values(PROPERTY_LABELS).some(l => b.includes(`**${l}:**`));
+				if (aKnown && !bKnown) return -1;
+				if (!aKnown && bKnown) return 1;
+				return a.localeCompare(b);
+			});
+
+			// Limit total properties shown
+			const maxProps = 50;
+			md += processedProperties.slice(0, maxProps).join("\n");
+			if (processedProperties.length > maxProps) {
+				md += `\n\n*...and ${processedProperties.length - maxProps} more properties*`;
+			}
+			md += "\n";
+		}
+
+		// Add notable sitelinks
+		if (entity.sitelinks) {
+			const notableSites = ["enwiki", "dewiki", "frwiki", "eswiki", "jawiki", "zhwiki"];
+			const links: string[] = [];
+
+			for (const site of notableSites) {
+				const sitelink = entity.sitelinks[site];
+				if (sitelink) {
+					const lang = site.replace("wiki", "");
+					const wikiUrl = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(sitelink.title)}`;
+					links.push(`[${lang.toUpperCase()}](${wikiUrl})`);
+				}
+			}
+
+			if (links.length > 0) {
+				md += `\n## Wikipedia Links\n\n${links.join(" · ")}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "wikidata",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Wikidata EntityData API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Get localized value with fallback
+ */
+function getLocalizedValue(
+	values: Record<string, { language: string; value: string }> | undefined,
+	preferredLang: string,
+): string | null {
+	if (!values) return null;
+	if (values[preferredLang]) return values[preferredLang].value;
+	// Fallback to any available
+	const first = Object.values(values)[0];
+	return first?.value || null;
+}
+
+/**
+ * Get aliases for a language
+ */
+function getLocalizedAliases(
+	aliases: Record<string, Array<{ language: string; value: string }>> | undefined,
+	preferredLang: string,
+): string[] {
+	if (!aliases) return [];
+	const langAliases = aliases[preferredLang];
+	if (!langAliases) return [];
+	return langAliases.map(a => a.value);
+}
+
+/**
+ * Resolve entity IDs to their labels via wbgetentities API
+ */
+async function resolveEntityLabels(
+	entityIds: string[],
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<Record<string, string>> {
+	if (entityIds.length === 0) return {};
+
+	const labels: Record<string, string> = {};
+
+	// Fetch in batches of 50
+	const batchSize = 50;
+	for (let i = 0; i < entityIds.length; i += batchSize) {
+		const batch = entityIds.slice(i, i + batchSize);
+		const apiUrl = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${batch.join("|")}&props=labels&languages=en&format=json`;
+
+		try {
+			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 10), signal });
+			if (result.ok) {
+				const data = JSON.parse(result.content) as {
+					entities: Record<string, { labels?: Record<string, { value: string }> }>;
+				};
+				for (const [id, entity] of Object.entries(data.entities)) {
+					const label = entity.labels?.en?.value;
+					if (label) labels[id] = label;
+				}
+			}
+		} catch {}
+	}
+
+	return labels;
+}
+
+/**
+ * Format a claim value to human-readable string
+ */
+function formatClaimValue(claim: WikidataClaim, entityLabels: Record<string, string>): string | null {
+	const snak = claim.mainsnak;
+	if (snak.snaktype !== "value" || !snak.datavalue) return null;
+
+	const { type, value } = snak.datavalue;
+
+	switch (type) {
+		case "wikibase-entityid": {
+			const entityVal = value as { id: string };
+			return entityLabels[entityVal.id] || entityVal.id;
+		}
+		case "string":
+			return value as string;
+		case "time": {
+			const timeVal = value as { time: string; precision: number };
+			return formatWikidataTime(timeVal.time, timeVal.precision);
+		}
+		case "quantity": {
+			const qtyVal = value as { amount: string; unit: string };
+			const amount = qtyVal.amount.replace(/^\+/, "");
+			// Extract unit Q-id if present
+			const unitMatch = qtyVal.unit.match(/Q\d+$/);
+			const unit = unitMatch ? entityLabels[unitMatch[0]] || "" : "";
+			return unit ? `${amount} ${unit}` : amount;
+		}
+		case "monolingualtext": {
+			const textVal = value as { text: string; language: string };
+			return textVal.text;
+		}
+		case "globecoordinate": {
+			const coordVal = value as { latitude: number; longitude: number };
+			return `${coordVal.latitude.toFixed(4)}, ${coordVal.longitude.toFixed(4)}`;
+		}
+		default:
+			return null;
+	}
+}
+
+/**
+ * Format Wikidata time value to readable date
+ */
+function formatWikidataTime(time: string, precision: number): string {
+	// Time format: +YYYY-MM-DDT00:00:00Z
+	const match = time.match(/^([+-]?\d+)-(\d{2})-(\d{2})/);
+	if (!match) return time;
+
+	const [, year, month, day] = match;
+	const yearNum = Number.parseInt(year, 10);
+	const absYear = Math.abs(yearNum);
+	const era = yearNum < 0 ? " BCE" : "";
+
+	// Precision: 9=year, 10=month, 11=day
+	if (precision >= 11) {
+		return `${day}/${month}/${absYear}${era}`;
+	}
+	if (precision >= 10) {
+		return `${month}/${absYear}${era}`;
+	}
+	return `${absYear}${era}`;
+}

package/src/web/scrapers/wikipedia.ts

@@ -0,0 +1,95 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Handle Wikipedia URLs via Wikipedia API
+ */
+export const handleWikipedia: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		// Match *.wikipedia.org
+		const wikiMatch = parsed.hostname.match(/^(\w+)\.wikipedia\.org$/);
+		if (!wikiMatch) return null;
+
+		const lang = wikiMatch[1];
+		const titleMatch = parsed.pathname.match(/\/wiki\/(.+)/);
+		if (!titleMatch) return null;
+
+		const title = decodeURIComponent(titleMatch[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Use Wikipedia API to get plain text extract
+		const apiUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`;
+		const summaryResult = await loadPage(apiUrl, { timeout, signal });
+
+		let md = "";
+
+		if (summaryResult.ok) {
+			const summary = JSON.parse(summaryResult.content) as {
+				title: string;
+				description?: string;
+				extract: string;
+			};
+			md = `# ${summary.title}\n\n`;
+			if (summary.description) md += `*${summary.description}*\n\n`;
+			md += `${summary.extract}\n\n---\n\n`;
+		}
+
+		// Get full article content via mobile-html or parse API
+		const contentUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/mobile-html/${encodeURIComponent(title)}`;
+		const contentResult = await loadPage(contentUrl, { timeout, signal });
+
+		if (contentResult.ok) {
+			const doc = parseHtml(contentResult.content);
+
+			// Extract main content sections
+			const sections = doc.querySelectorAll("section");
+			for (const section of sections) {
+				const heading = section.querySelector("h2, h3, h4");
+				const headingText = heading?.text?.trim();
+
+				// Skip certain sections
+				if (
+					headingText &&
+					["References", "External links", "See also", "Notes", "Further reading"].includes(headingText)
+				) {
+					continue;
+				}
+
+				if (headingText) {
+					const level = heading?.tagName === "H2" ? "##" : "###";
+					md += `${level} ${headingText}\n\n`;
+				}
+
+				const paragraphs = section.querySelectorAll("p");
+				for (const p of paragraphs) {
+					const text = p.text?.trim();
+					if (text && text.length > 20) {
+						md += `${text}\n\n`;
+					}
+				}
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "wikipedia",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Wikipedia API"],
+		};
+	} catch {}
+
+	return null;
+};