@nghyane/arcane 0.1.0

package/src/web/scrapers/sourcegraph.ts

@@ -0,0 +1,373 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const GRAPHQL_ENDPOINT = "https://sourcegraph.com/.api/graphql";
+const GRAPHQL_HEADERS = {
+	Accept: "application/json",
+	"Content-Type": "application/json",
+};
+
+type SourcegraphTarget =
+	| { type: "search"; query: string }
+	| { type: "repo"; repoName: string; rev?: string }
+	| { type: "file"; repoName: string; rev?: string; filePath: string };
+
+interface SourcegraphRepository {
+	name: string;
+	url: string;
+	description?: string | null;
+	defaultBranch?: { name: string } | null;
+}
+
+interface RepoQueryData {
+	repository?: SourcegraphRepository | null;
+}
+
+interface RepoFileQueryData {
+	repository?:
+		| (SourcegraphRepository & {
+				commit?: {
+					blob?: { content?: string | null } | null;
+				} | null;
+		  })
+		| null;
+}
+
+interface SearchQueryData {
+	search?: {
+		results?: {
+			results?: SearchResultItem[] | null;
+			matchCount?: number | null;
+			limitHit?: boolean | null;
+		} | null;
+	} | null;
+}
+
+interface FileMatchResult {
+	__typename: "FileMatch";
+	repository?: { name?: string | null; url?: string | null } | null;
+	file?: { path?: string | null; url?: string | null } | null;
+	lineMatches?: Array<{ preview?: string | null; lineNumber?: number | null }> | null;
+}
+
+interface RepositoryResult {
+	__typename: "Repository";
+	name?: string | null;
+	url?: string | null;
+}
+
+type SearchResultItem = FileMatchResult | RepositoryResult | { __typename: string };
+
+const REPO_QUERY = `query Repo($name: String!) {
+	repository(name: $name) {
+		name
+		url
+		description
+		defaultBranch {
+			name
+		}
+	}
+}`;
+
+const REPO_FILE_QUERY = `query RepoFile($name: String!, $path: String!, $rev: String!) {
+	repository(name: $name) {
+		name
+		url
+		description
+		defaultBranch {
+			name
+		}
+		commit(rev: $rev) {
+			blob(path: $path) {
+				content
+			}
+		}
+	}
+}`;
+
+const SEARCH_QUERY = `query Search($query: String!) {
+	search(query: $query, version: V2) {
+		results {
+			results {
+				__typename
+				... on FileMatch {
+					repository {
+						name
+						url
+					}
+					file {
+						path
+						url
+					}
+					lineMatches {
+						preview
+						lineNumber
+					}
+				}
+				... on Repository {
+					name
+					url
+				}
+			}
+			matchCount
+			limitHit
+		}
+	}
+}`;
+
+function parseSourcegraphUrl(url: string): SourcegraphTarget | null {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "sourcegraph.com" && parsed.hostname !== "www.sourcegraph.com") return null;
+
+		if (parsed.pathname.startsWith("/search")) {
+			const query = parsed.searchParams.get("q")?.trim();
+			if (!query) return null;
+			return { type: "search", query };
+		}
+
+		const parts = parsed.pathname
+			.split("/")
+			.filter(Boolean)
+			.map(part => decodeURIComponent(part));
+		if (parts.length < 3) return null;
+
+		const hyphenIndex = parts.indexOf("-");
+		const repoParts = hyphenIndex === -1 ? parts : parts.slice(0, hyphenIndex);
+		if (repoParts.length < 3) return null;
+
+		const lastRepoPart = repoParts[repoParts.length - 1];
+		const atIndex = lastRepoPart.indexOf("@");
+		let rev: string | undefined;
+		let repoTail = lastRepoPart;
+		if (atIndex > 0) {
+			repoTail = lastRepoPart.slice(0, atIndex);
+			rev = lastRepoPart.slice(atIndex + 1) || undefined;
+		}
+
+		repoParts[repoParts.length - 1] = repoTail;
+		const repoName = repoParts.join("/");
+
+		if (hyphenIndex !== -1 && parts[hyphenIndex + 1] === "blob") {
+			const filePath = parts.slice(hyphenIndex + 2).join("/");
+			if (!filePath) return null;
+			return { type: "file", repoName, rev, filePath };
+		}
+
+		return { type: "repo", repoName, rev };
+	} catch {
+		return null;
+	}
+}
+
+function safeParseJson<T>(content: string): T | null {
+	try {
+		return JSON.parse(content) as T;
+	} catch {
+		return null;
+	}
+}
+
+async function fetchGraphql<T>(
+	query: string,
+	variables: Record<string, unknown>,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<T | null> {
+	const body = JSON.stringify({ query, variables });
+	const result = await loadPage(GRAPHQL_ENDPOINT, {
+		timeout,
+		headers: GRAPHQL_HEADERS,
+		method: "POST",
+		body,
+		signal,
+	});
+	if (!result.ok) return null;
+
+	const parsed = safeParseJson<{ data?: T; errors?: unknown }>(result.content);
+	if (!parsed?.data) return null;
+	if (Array.isArray(parsed.errors) && parsed.errors.length > 0) return null;
+	return parsed.data;
+}
+
+function isFileMatchResult(result: SearchResultItem): result is FileMatchResult {
+	return result.__typename === "FileMatch";
+}
+
+function isRepositoryResult(result: SearchResultItem): result is RepositoryResult {
+	return result.__typename === "Repository";
+}
+
+function formatRepoMarkdown(repo: SourcegraphRepository): string {
+	let md = `# ${repo.name}\n\n`;
+	if (repo.description) md += `${repo.description}\n\n`;
+	md += `**URL:** ${repo.url}\n`;
+	if (repo.defaultBranch?.name) md += `**Default branch:** ${repo.defaultBranch.name}\n`;
+	return md;
+}
+
+async function renderRepo(
+	repoName: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<RepoQueryData>(REPO_QUERY, { name: repoName }, timeout, signal);
+	if (!data?.repository) return { content: "", ok: false };
+
+	return { content: formatRepoMarkdown(data.repository), ok: true };
+}
+
+async function renderFile(
+	repoName: string,
+	filePath: string,
+	rev: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<RepoFileQueryData>(
+		REPO_FILE_QUERY,
+		{ name: repoName, path: filePath, rev },
+		timeout,
+		signal,
+	);
+	const repo = data?.repository;
+	const content = repo?.commit?.blob?.content ?? null;
+	if (!repo || content === null) return { content: "", ok: false };
+
+	let md = `${formatRepoMarkdown(repo)}\n`;
+	md += `**Path:** ${filePath}\n`;
+	md += `**Revision:** ${rev}\n\n`;
+	md += `---\n\n## File\n\n`;
+	md += "```text\n";
+	md += `${content}\n`;
+	md += "```\n";
+	return { content: md, ok: true };
+}
+
+async function renderSearch(
+	query: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<SearchQueryData>(SEARCH_QUERY, { query }, timeout, signal);
+	const resultsData = data?.search?.results;
+	if (!resultsData) return { content: "", ok: false };
+	const results = resultsData.results ?? [];
+
+	let md = "# Sourcegraph Search\n\n";
+	md += `**Query:** \`${query}\`\n`;
+	if (typeof resultsData?.matchCount === "number") {
+		md += `**Matches:** ${resultsData.matchCount}\n`;
+	}
+	if (typeof resultsData?.limitHit === "boolean") {
+		md += `**Limit hit:** ${resultsData.limitHit ? "yes" : "no"}\n`;
+	}
+	md += "\n";
+
+	if (!results || results.length === 0) {
+		md += "_No results._\n";
+		return { content: md, ok: true };
+	}
+
+	const maxResults = 10;
+	md += "## Results\n\n";
+	for (const result of results.slice(0, maxResults)) {
+		if (isFileMatchResult(result)) {
+			const repoName = result.repository?.name ?? "unknown";
+			const filePath = result.file?.path ?? "unknown";
+			md += `### ${repoName}/${filePath}\n\n`;
+			if (result.repository?.url) md += `**Repository:** ${result.repository.url}\n`;
+			if (result.file?.url) md += `**File:** ${result.file.url}\n`;
+
+			const lineMatches = result.lineMatches ?? [];
+			if (lineMatches.length > 0) {
+				md += "\n```text\n";
+				for (const line of lineMatches.slice(0, 5)) {
+					const preview = (line.preview ?? "").replace(/\n/g, " ").trim();
+					const lineNumber = line.lineNumber ?? 0;
+					md += `L${lineNumber}: ${preview}\n`;
+				}
+				md += "```\n\n";
+			}
+			continue;
+		}
+
+		if (isRepositoryResult(result)) {
+			const name = result.name ?? "unknown";
+			md += `### ${name}\n\n`;
+			if (result.url) md += `**Repository:** ${result.url}\n`;
+			md += "\n";
+		}
+	}
+
+	if (results.length > maxResults) {
+		md += `... and ${results.length - maxResults} more results\n`;
+	}
+
+	return { content: md, ok: true };
+}
+
+export const handleSourcegraph: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const target = parseSourcegraphUrl(url);
+		if (!target) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes = ["Fetched via Sourcegraph GraphQL API"];
+
+		switch (target.type) {
+			case "search": {
+				const result = await renderSearch(target.query, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-search",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+			case "file": {
+				const rev = target.rev ?? "HEAD";
+				const result = await renderFile(target.repoName, target.filePath, rev, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-file",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+			case "repo": {
+				const result = await renderRepo(target.repoName, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-repo",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+		}
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/spdx.ts

@@ -0,0 +1,121 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface SpdxCrossRef {
+	url?: string;
+	isValid?: boolean;
+	isLive?: boolean;
+	match?: string;
+	order?: number;
+}
+
+interface SpdxLicense {
+	licenseId: string;
+	name: string;
+	isOsiApproved?: boolean;
+	isFsfLibre?: boolean;
+	licenseText?: string;
+	licenseTextHtml?: string;
+	seeAlso?: string[];
+	crossRef?: SpdxCrossRef[];
+	comment?: string;
+	licenseComments?: string;
+}
+
+function formatYesNo(value?: boolean): string {
+	if (value === true) return "Yes";
+	if (value === false) return "No";
+	return "Unknown";
+}
+
+function collectCrossReferences(license: SpdxLicense): string[] {
+	const ordered = (license.crossRef ?? [])
+		.filter(ref => ref.url)
+		.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
+		.map(ref => ref.url as string);
+
+	const seeAlso = (license.seeAlso ?? []).filter(url => url);
+	const combined = [...ordered, ...seeAlso];
+	return combined.filter((url, index) => combined.indexOf(url) === index);
+}
+
+/**
+ * Handle SPDX license URLs via SPDX JSON API
+ */
+export const handleSpdx: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "spdx.org" && parsed.hostname !== "www.spdx.org") return null;
+
+		const match = parsed.pathname.match(/^\/licenses\/([^/]+?)(?:\.html)?\/?$/i);
+		if (!match) return null;
+
+		const licenseId = decodeURIComponent(match[1]);
+		if (!licenseId) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://spdx.org/licenses/${encodeURIComponent(licenseId)}.json`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let license: SpdxLicense;
+		try {
+			license = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const title = license.name || license.licenseId || licenseId;
+		let md = `# ${title}\n\n`;
+
+		md += `**License ID:** ${license.licenseId ? `\`${license.licenseId}\`` : `\`${licenseId}\``}\n`;
+		md += `**OSI Approved:** ${formatYesNo(license.isOsiApproved)}\n`;
+		md += `**FSF Libre:** ${formatYesNo(license.isFsfLibre)}\n`;
+
+		const description = license.licenseComments ?? license.comment;
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		const crossReferences = collectCrossReferences(license);
+		if (crossReferences.length) {
+			md += `\n## Cross References\n\n`;
+			for (const ref of crossReferences) {
+				md += `- ${ref}\n`;
+			}
+		}
+
+		const licenseText = license.licenseText
+			? license.licenseText
+			: license.licenseTextHtml
+				? htmlToBasicMarkdown(license.licenseTextHtml)
+				: null;
+
+		if (licenseText) {
+			md += `\n## License Text\n\n\`\`\`\n${licenseText}\n\`\`\`\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "spdx-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via SPDX license API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/spotify.ts

@@ -0,0 +1,217 @@
+/**
+ * Spotify URL handler for podcasts, tracks, albums, and playlists
+ *
+ * Uses oEmbed API and Open Graph metadata to extract information
+ * from Spotify URLs without requiring authentication.
+ */
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface SpotifyOEmbedResponse {
+	title?: string;
+	thumbnail_url?: string;
+	provider_name?: string;
+	html?: string;
+	width?: number;
+	height?: number;
+}
+
+interface OpenGraphData {
+	title?: string;
+	description?: string;
+	audio?: string;
+	image?: string;
+	type?: string;
+	duration?: string;
+	album?: string;
+	musician?: string;
+	artist?: string;
+	releaseDate?: string;
+}
+
+/**
+ * Parse Open Graph meta tags from HTML
+ */
+function parseOpenGraph(html: string): OpenGraphData {
+	const og: OpenGraphData = {};
+
+	const metaPattern = /<meta\s+(?:property|name)="([^"]+)"\s+content="([^"]*)"[^>]*>/gi;
+	let match: RegExpExecArray | null = null;
+
+	while (true) {
+		match = metaPattern.exec(html);
+		if (match === null) break;
+		const [, property, content] = match;
+
+		if (property === "og:title") og.title = content;
+		else if (property === "og:description") og.description = content;
+		else if (property === "og:audio") og.audio = content;
+		else if (property === "og:image") og.image = content;
+		else if (property === "og:type") og.type = content;
+		else if (property === "music:duration") og.duration = content;
+		else if (property === "music:album") og.album = content;
+		else if (property === "music:musician") og.musician = content;
+		else if (property === "music:release_date") og.releaseDate = content;
+		else if (property === "twitter:audio:artist_name") og.artist = content;
+	}
+
+	return og;
+}
+
+/**
+ * Determine content type from URL path
+ */
+function getContentType(url: string): string | null {
+	if (url.includes("/episode/")) return "podcast-episode";
+	if (url.includes("/show/")) return "podcast-show";
+	if (url.includes("/track/")) return "track";
+	if (url.includes("/album/")) return "album";
+	if (url.includes("/playlist/")) return "playlist";
+	return null;
+}
+
+/**
+ * Format duration from seconds
+ */
+function formatDuration(seconds: string | undefined): string | null {
+	if (!seconds) return null;
+	const num = parseInt(seconds, 10);
+	if (Number.isNaN(num)) return null;
+
+	const hours = Math.floor(num / 3600);
+	const minutes = Math.floor((num % 3600) / 60);
+	const secs = num % 60;
+
+	if (hours > 0) {
+		return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
+	}
+	return `${minutes}:${secs.toString().padStart(2, "0")}`;
+}
+
+/**
+ * Format output based on content type and available metadata
+ */
+function formatOutput(contentType: string, oEmbed: SpotifyOEmbedResponse, og: OpenGraphData, url: string): string {
+	const sections: string[] = [];
+
+	// Title
+	const title = og.title || oEmbed.title || "Unknown";
+	sections.push(`# ${title}\n`);
+
+	// Type
+	sections.push(`**Type**: ${contentType}\n`);
+
+	// Description
+	if (og.description) {
+		sections.push(`**Description**: ${og.description}\n`);
+	}
+
+	// Content-specific metadata
+	if (contentType === "track" || contentType === "podcast-episode") {
+		if (og.artist || og.musician) {
+			sections.push(`**Artist**: ${og.artist || og.musician}\n`);
+		}
+		if (og.album) {
+			sections.push(`**Album**: ${og.album}\n`);
+		}
+		if (og.duration) {
+			const formatted = formatDuration(og.duration);
+			if (formatted) {
+				sections.push(`**Duration**: ${formatted}\n`);
+			}
+		}
+	}
+
+	if (contentType === "album" && og.releaseDate) {
+		sections.push(`**Release Date**: ${og.releaseDate}\n`);
+	}
+
+	// Note about limited information
+	sections.push("\n---\n");
+	if (contentType === "playlist") {
+		sections.push(
+			"**Note**: Playlist details (tracks, creator, follower count) require authentication. " +
+				"Only basic metadata is available without Spotify API credentials.\n",
+		);
+	} else if (contentType === "album") {
+		sections.push(
+			"**Note**: Track listing and detailed album information require authentication. " +
+				"Only basic metadata is available without Spotify API credentials.\n",
+		);
+	} else if (contentType === "podcast-show") {
+		sections.push(
+			"**Note**: Episode listing and detailed show information require authentication. " +
+				"Only basic metadata is available without Spotify API credentials.\n",
+		);
+	}
+
+	sections.push(`**URL**: ${url}\n`);
+
+	if (oEmbed.thumbnail_url) {
+		sections.push(`**Thumbnail**: ${oEmbed.thumbnail_url}\n`);
+	} else if (og.image) {
+		sections.push(`**Image**: ${og.image}\n`);
+	}
+
+	return sections.join("\n");
+}
+
+export const handleSpotify: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	// Check if this is a Spotify URL
+	if (!url.includes("open.spotify.com/")) {
+		return null;
+	}
+
+	const contentType = getContentType(url);
+	if (!contentType) {
+		return null;
+	}
+
+	const notes: string[] = [];
+	let oEmbedData: SpotifyOEmbedResponse = {};
+	let ogData: OpenGraphData = {};
+
+	// Fetch oEmbed data
+	try {
+		const oEmbedUrl = `https://open.spotify.com/oembed?url=${encodeURIComponent(url)}`;
+		const response = await loadPage(oEmbedUrl, { timeout, signal });
+
+		if (response.ok) {
+			oEmbedData = JSON.parse(response.content) as SpotifyOEmbedResponse;
+			notes.push("Retrieved metadata via Spotify oEmbed API");
+		} else {
+			notes.push(`oEmbed API returned status ${response.status || "error"}`);
+		}
+	} catch (err) {
+		notes.push(`Failed to fetch oEmbed data: ${err instanceof Error ? err.message : String(err)}`);
+	}
+
+	// Fetch page HTML for Open Graph metadata
+	try {
+		const pageResponse = await loadPage(url, { timeout, signal });
+
+		if (pageResponse.ok) {
+			ogData = parseOpenGraph(pageResponse.content);
+			notes.push("Parsed Open Graph metadata from page HTML");
+		} else {
+			notes.push(`Page fetch returned status ${pageResponse.status || "error"}`);
+		}
+	} catch (err) {
+		notes.push(`Failed to fetch page HTML: ${err instanceof Error ? err.message : String(err)}`);
+	}
+
+	// Format output
+	const output = formatOutput(contentType, oEmbedData, ogData, url);
+	const { content, truncated } = finalizeOutput(output);
+
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "spotify",
+		content,
+		fetchedAt: new Date().toISOString(),
+		truncated,
+		notes,
+	};
+};