@nghyane/arcane 0.1.0

package/src/web/scrapers/types.ts

@@ -0,0 +1,200 @@
+/**
+ * Shared types and utilities for web-fetch handlers
+ */
+import { ptree } from "@nghyane/arcane-utils";
+import { ToolAbortError } from "../../tools/tool-errors";
+
+export interface RenderResult {
+	url: string;
+	finalUrl: string;
+	contentType: string;
+	method: string;
+	content: string;
+	fetchedAt: string;
+	truncated: boolean;
+	notes: string[];
+}
+
+export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal) => Promise<RenderResult | null>;
+
+export const MAX_OUTPUT_CHARS = 500_000;
+const MAX_BYTES = 50 * 1024 * 1024;
+
+const USER_AGENTS = [
+	"curl/8.0",
+	"Mozilla/5.0 (compatible; TextBot/1.0)",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+];
+
+function isBotBlocked(status: number, content: string): boolean {
+	if (status === 403 || status === 503) {
+		const lower = content.toLowerCase();
+		return (
+			lower.includes("cloudflare") ||
+			lower.includes("captcha") ||
+			lower.includes("challenge") ||
+			lower.includes("blocked") ||
+			lower.includes("access denied") ||
+			lower.includes("bot detection")
+		);
+	}
+	return false;
+}
+
+/**
+ * Truncate and cleanup output
+ */
+export function finalizeOutput(content: string): { content: string; truncated: boolean } {
+	const cleaned = content.replace(/\n{3,}/g, "\n\n").trim();
+	const truncated = cleaned.length > MAX_OUTPUT_CHARS;
+	return {
+		content: cleaned.slice(0, MAX_OUTPUT_CHARS),
+		truncated,
+	};
+}
+
+export interface LoadPageOptions {
+	timeout?: number;
+	headers?: Record<string, string>;
+	method?: string;
+	body?: string;
+	maxBytes?: number;
+	signal?: AbortSignal;
+}
+
+export interface LoadPageResult {
+	content: string;
+	contentType: string;
+	finalUrl: string;
+	ok: boolean;
+	status?: number;
+}
+
+/**
+ * Fetch a page with timeout and size limit
+ */
+export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
+	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
+
+	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
+		if (signal?.aborted) {
+			throw new ToolAbortError();
+		}
+
+		const userAgent = USER_AGENTS[attempt];
+		const requestSignal = ptree.combineSignals(signal, timeout * 1000);
+
+		try {
+			const requestInit: RequestInit = {
+				signal: requestSignal,
+				method,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					"Accept-Encoding": "identity", // Cloudflare Markdown-for-Agents returns corrupted bytes when compression is negotiated
+					...headers,
+				},
+				redirect: "follow",
+			};
+
+			if (body !== undefined) {
+				requestInit.body = body;
+			}
+
+			const response = await fetch(url, requestInit);
+
+			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const finalUrl = response.url;
+
+			const reader = response.body?.getReader();
+			if (!reader) {
+				return { content: "", contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			const chunks: Uint8Array[] = [];
+			let totalSize = 0;
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				chunks.push(value);
+				totalSize += value.length;
+
+				if (totalSize > maxBytes) {
+					reader.cancel();
+					break;
+				}
+			}
+
+			const content = Buffer.concat(chunks).toString("utf-8");
+			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
+				continue;
+			}
+
+			if (!response.ok) {
+				return { content, contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			return { content, contentType, finalUrl, ok: true, status: response.status };
+		} catch {
+			if (signal?.aborted) {
+				throw new ToolAbortError();
+			}
+			if (attempt === USER_AGENTS.length - 1) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+		}
+	}
+
+	return { content: "", contentType: "", finalUrl: url, ok: false };
+}
+
+/**
+ * Format large numbers (1000 -> 1K, 1000000 -> 1M)
+ */
+export function formatCount(n: number): string {
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+	if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+	return String(n);
+}
+
+/**
+ * Convert basic HTML to markdown
+ */
+export function htmlToBasicMarkdown(html: string): string {
+	return html
+		.replace(/<pre><code[^>]*>/g, "\n```\n")
+		.replace(/<\/code><\/pre>/g, "\n```\n")
+		.replace(/<code>/g, "`")
+		.replace(/<\/code>/g, "`")
+		.replace(/<strong>/g, "**")
+		.replace(/<\/strong>/g, "**")
+		.replace(/<b>/g, "**")
+		.replace(/<\/b>/g, "**")
+		.replace(/<em>/g, "*")
+		.replace(/<\/em>/g, "*")
+		.replace(/<i>/g, "*")
+		.replace(/<\/i>/g, "*")
+		.replace(/<a href="([^"]+)"[^>]*>([^<]+)<\/a>/g, "[$2]($1)")
+		.replace(/<p>/g, "\n\n")
+		.replace(/<\/p>/g, "")
+		.replace(/<br\s*\/?>/g, "\n")
+		.replace(/<li>/g, "- ")
+		.replace(/<\/li>/g, "\n")
+		.replace(/<\/?[uo]l>/g, "\n")
+		.replace(/<h(\d)>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
+		.replace(/<\/h\d>/g, "\n")
+		.replace(/<blockquote>/g, "\n> ")
+		.replace(/<\/blockquote>/g, "\n")
+		.replace(/<[^>]+>/g, "")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&nbsp;/g, " ")
+		.replace(/\n{3,}/g, "\n\n")
+		.trim();
+}

package/src/web/scrapers/utils.ts

@@ -0,0 +1,142 @@
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
+import { ptree, Snowflake } from "@nghyane/arcane-utils";
+import { ensureTool } from "../../utils/tools-manager";
+
+const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files
+
+export interface ConvertResult {
+	content: string;
+	ok: boolean;
+	error?: string;
+}
+
+export interface BinaryFetchResult {
+	buffer: Buffer;
+	contentType: string;
+	contentDisposition?: string;
+	ok: boolean;
+	status?: number;
+	error?: string;
+}
+
+export async function convertWithMarkitdown(
+	content: Buffer,
+	extensionHint: string,
+	timeout: number,
+	userSignal?: AbortSignal,
+): Promise<ConvertResult> {
+	if (userSignal?.aborted) {
+		return { content: "", ok: false, error: "aborted" };
+	}
+
+	const signal = ptree.combineSignals(userSignal, timeout * 1000);
+
+	const markitdown = await ensureTool("markitdown", { signal, silent: true });
+	if (!markitdown) {
+		return { content: "", ok: false, error: "markitdown not available" };
+	}
+
+	// Write to temp file with extension hint
+	const ext = extensionHint || ".bin";
+	const tmpDir = os.tmpdir();
+	const tmpFile = path.join(tmpDir, `arc-convert-${Snowflake.next()}${ext}`);
+
+	if (content.length > MAX_BYTES) {
+		return { content: "", ok: false, error: `content exceeds ${MAX_BYTES} bytes` };
+	}
+
+	try {
+		await Bun.write(tmpFile, content);
+		const result = await ptree.exec([markitdown, tmpFile], {
+			signal,
+			allowNonZero: true,
+			stderr: "full",
+		});
+		if (!result.ok) {
+			return {
+				content: result.stdout,
+				ok: false,
+				error:
+					result.stderr.length > 0 ? result.stderr : `markitdown failed (exit ${result.exitCode ?? "unknown"})`,
+			};
+		}
+		return { content: result.stdout, ok: true };
+	} finally {
+		void fs.rm(tmpFile, { force: true }).catch(() => {});
+	}
+}
+
+const kEmptyBuffer = Buffer.alloc(0);
+
+export async function fetchBinary(url: string, timeout: number, userSignal?: AbortSignal): Promise<BinaryFetchResult> {
+	if (userSignal?.aborted) {
+		return { buffer: kEmptyBuffer, contentType: "", ok: false, error: "aborted" };
+	}
+
+	const signal = ptree.combineSignals(userSignal, timeout * 1000);
+
+	try {
+		const response = await fetch(url, {
+			signal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0",
+			},
+			redirect: "follow",
+		});
+
+		const contentType = response.headers.get("content-type") ?? "";
+		const contentDisposition = response.headers.get("content-disposition") ?? undefined;
+
+		if (!response.ok) {
+			return {
+				buffer: kEmptyBuffer,
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `status ${response.status}`,
+			};
+		}
+
+		const contentLength = response.headers.get("content-length");
+		if (contentLength) {
+			const size = Number.parseInt(contentLength, 10);
+			if (Number.isFinite(size) && size > MAX_BYTES) {
+				return {
+					buffer: kEmptyBuffer,
+					contentType,
+					contentDisposition,
+					ok: false,
+					status: response.status,
+					error: `content-length ${size} exceeds ${MAX_BYTES}`,
+				};
+			}
+		}
+
+		const buffer = Buffer.from(await response.arrayBuffer());
+		if (buffer.length > MAX_BYTES) {
+			return {
+				buffer: kEmptyBuffer,
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `response exceeds ${MAX_BYTES} bytes`,
+			};
+		}
+
+		return { buffer, contentType, contentDisposition, ok: true, status: response.status };
+	} catch (err) {
+		if (signal?.aborted) {
+			return { buffer: kEmptyBuffer, contentType: "", ok: false, error: "aborted" };
+		}
+		return {
+			buffer: kEmptyBuffer,
+			contentType: "",
+			ok: false,
+			error: `request failed: ${String(err)}`,
+		};
+	}
+}

package/src/web/scrapers/vimeo.ts

@@ -0,0 +1,152 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface VimeoOEmbed {
+	title: string;
+	author_name: string;
+	author_url: string;
+	description?: string;
+	duration: number;
+	thumbnail_url: string;
+	upload_date: string;
+	video_id: number;
+}
+
+interface VimeoVideoConfig {
+	video?: {
+		title?: string;
+		duration?: number;
+		owner?: {
+			name?: string;
+			url?: string;
+		};
+		thumbs?: {
+			base?: string;
+		};
+	};
+	request?: {
+		files?: {
+			progressive?: Array<{
+				quality: string;
+				width: number;
+				height: number;
+				fps: number;
+			}>;
+		};
+	};
+}
+
+/**
+ * Format seconds into HH:MM:SS or MM:SS
+ */
+function formatDuration(seconds: number): string {
+	const h = Math.floor(seconds / 3600);
+	const m = Math.floor((seconds % 3600) / 60);
+	const s = seconds % 60;
+	if (h > 0) return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
+	return `${m}:${String(s).padStart(2, "0")}`;
+}
+
+/**
+ * Extract video ID from various Vimeo URL formats
+ */
+function extractVideoId(url: string): string | null {
+	try {
+		const parsed = new URL(url);
+
+		// player.vimeo.com/video/{id}
+		if (parsed.hostname === "player.vimeo.com") {
+			const match = parsed.pathname.match(/^\/video\/(\d+)/);
+			return match?.[1] ?? null;
+		}
+
+		// vimeo.com/{id} or vimeo.com/{user}/{id}
+		if (parsed.hostname === "vimeo.com" || parsed.hostname === "www.vimeo.com") {
+			const parts = parsed.pathname.split("/").filter(Boolean);
+			// Last part should be the video ID
+			const lastPart = parts[parts.length - 1];
+			if (lastPart && /^\d+$/.test(lastPart)) {
+				return lastPart;
+			}
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Handle Vimeo URLs via oEmbed API
+ */
+export const handleVimeo: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("vimeo.com")) return null;
+
+		const videoId = extractVideoId(url);
+		if (!videoId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Use canonical URL for oEmbed (handles staffpicks and other URL formats)
+		const canonicalUrl = `https://vimeo.com/${videoId}`;
+		const oembedUrl = `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(canonicalUrl)}`;
+		const oembedResult = await loadPage(oembedUrl, { timeout, signal });
+
+		if (!oembedResult.ok) return null;
+
+		const oembed = JSON.parse(oembedResult.content) as VimeoOEmbed;
+
+		let md = `# ${oembed.title}\n\n`;
+		md += `**Author:** [${oembed.author_name}](${oembed.author_url})\n`;
+		md += `**Duration:** ${formatDuration(oembed.duration)}\n`;
+
+		if (oembed.upload_date) {
+			md += `**Uploaded:** ${oembed.upload_date}\n`;
+		}
+
+		md += `**Video ID:** ${videoId}\n\n`;
+
+		if (oembed.description) {
+			md += `---\n\n## Description\n\n${oembed.description}\n\n`;
+		}
+
+		md += `---\n\n**Thumbnail:** ${oembed.thumbnail_url}\n`;
+
+		// Try to get additional details from video config
+		try {
+			const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
+			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (configResult.ok) {
+				const config = JSON.parse(configResult.content) as VimeoVideoConfig;
+
+				// Add video quality info if available
+				const progressive = config.request?.files?.progressive;
+				if (progressive && progressive.length > 0) {
+					md += `\n**Available Qualities:**\n`;
+					for (const quality of progressive.slice(0, 5)) {
+						md += `- ${quality.quality}: ${quality.width}x${quality.height} @ ${quality.fps}fps\n`;
+					}
+				}
+			}
+		} catch {
+			// Config fetch is optional - continue without it
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "vimeo",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Vimeo oEmbed API"],
+		};
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/vscode-marketplace.ts

@@ -0,0 +1,195 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface MarketplaceProperty {
+	key?: string;
+	value?: string;
+}
+
+interface MarketplaceVersion {
+	version?: string;
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceStatistic {
+	statisticName?: string;
+	value?: number;
+}
+
+interface MarketplacePublisher {
+	publisherName?: string;
+	displayName?: string;
+}
+
+interface MarketplaceExtension {
+	extensionName?: string;
+	displayName?: string;
+	shortDescription?: string;
+	description?: string;
+	publisher?: MarketplacePublisher;
+	versions?: MarketplaceVersion[];
+	statistics?: MarketplaceStatistic[];
+	categories?: string[];
+	tags?: string[];
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceResponse {
+	results?: Array<{ extensions?: MarketplaceExtension[] }>;
+}
+
+const MARKETPLACE_HOSTS = new Set(["marketplace.visualstudio.com", "www.marketplace.visualstudio.com"]);
+
+function getItemName(parsed: URL): string | null {
+	if (!parsed.pathname.startsWith("/items")) return null;
+	const itemName = parsed.searchParams.get("itemName");
+	if (!itemName) return null;
+	const decoded = decodeURIComponent(itemName);
+	if (!decoded.includes(".")) return null;
+	return decoded;
+}
+
+function toStatMap(stats: MarketplaceStatistic[] | undefined): Map<string, number> {
+	const map = new Map<string, number>();
+	if (!stats) return map;
+	for (const stat of stats) {
+		if (!stat.statisticName || typeof stat.value !== "number") continue;
+		map.set(stat.statisticName.trim().toLowerCase(), stat.value);
+	}
+	return map;
+}
+
+function formatRating(averageRating?: number, ratingCount?: number): string | null {
+	if (averageRating === undefined && ratingCount === undefined) return null;
+	if (averageRating !== undefined) {
+		const formatted = averageRating.toFixed(2).replace(/\.0+$/, "").replace(/\.$/, "");
+		if (ratingCount !== undefined) {
+			return `${formatted} (${formatCount(ratingCount)} ratings)`;
+		}
+		return formatted;
+	}
+	if (ratingCount !== undefined) {
+		return `${formatCount(ratingCount)} ratings`;
+	}
+	return null;
+}
+
+function extractRepoLink(properties: MarketplaceProperty[] | undefined): string | null {
+	if (!properties) return null;
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key.includes("links.source") || key.includes("repository")) return value;
+	}
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key === "source" || key.endsWith(".source")) return value;
+	}
+	return null;
+}
+
+/**
+ * Handle VS Code Marketplace URLs via extension query API
+ */
+export const handleVscodeMarketplace: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null;
+
+		const itemName = getItemName(parsed);
+		if (!itemName) return null;
+
+		const [publisherFromUrl, ...nameParts] = itemName.split(".");
+		const extensionFromUrl = nameParts.join(".");
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery";
+		const payload = JSON.stringify({
+			filters: [
+				{
+					criteria: [{ filterType: 7, value: itemName }],
+				},
+			],
+			flags: 950,
+		});
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			method: "POST",
+			body: payload,
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json;api-version=7.2-preview.1",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let data: MarketplaceResponse;
+		try {
+			data = JSON.parse(result.content) as MarketplaceResponse;
+		} catch {
+			return null;
+		}
+
+		const extension = data.results?.[0]?.extensions?.[0];
+		if (!extension) return null;
+
+		const extensionName = extension.extensionName ?? extensionFromUrl;
+		const displayName = extension.displayName ?? extensionName ?? itemName;
+		const description = extension.shortDescription ?? extension.description;
+
+		const publisherName = extension.publisher?.publisherName ?? publisherFromUrl;
+		const publisherDisplayName = extension.publisher?.displayName;
+		const publisherLabel =
+			publisherDisplayName && publisherName && publisherDisplayName !== publisherName
+				? `${publisherDisplayName} (${publisherName})`
+				: (publisherDisplayName ?? publisherName);
+
+		const version = extension.versions?.[0]?.version;
+		const statMap = toStatMap(extension.statistics);
+		const installs = statMap.get("install") ?? statMap.get("installs");
+		const averageRating = statMap.get("averagerating");
+		const ratingCount = statMap.get("ratingcount");
+		const ratingLabel = formatRating(averageRating, ratingCount);
+
+		const repoLink = extractRepoLink(extension.versions?.[0]?.properties) ?? extractRepoLink(extension.properties);
+
+		const identifier = publisherName && extensionName ? `${publisherName}.${extensionName}` : itemName;
+
+		let md = `# ${displayName}\n\n`;
+		if (description) md += `${description}\n\n`;
+		md += `**Identifier:** ${identifier}\n`;
+		if (publisherLabel) md += `**Publisher:** ${publisherLabel}\n`;
+		if (version) md += `**Version:** ${version}\n`;
+		if (installs !== undefined) md += `**Installs:** ${formatCount(installs)}\n`;
+		if (ratingLabel) md += `**Rating:** ${ratingLabel}\n`;
+		if (extension.categories?.length) md += `**Categories:** ${extension.categories.join(", ")}\n`;
+		if (extension.tags?.length) md += `**Tags:** ${extension.tags.join(", ")}\n`;
+		if (repoLink) md += `**Repository:** ${repoLink}\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "vscode-marketplace",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via VS Code Marketplace API"],
+		};
+	} catch {}
+
+	return null;
+};