@nghyane/arcane 0.1.0

package/src/web/scrapers/youtube.ts

@@ -0,0 +1,312 @@
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import path from "node:path";
+import { ptree, Snowflake } from "@nghyane/arcane-utils";
+import { throwIfAborted } from "../../tools/tool-errors";
+import { ensureTool } from "../../utils/tools-manager";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput } from "./types";
+
+interface YouTubeUrl {
+	videoId: string;
+	playlistId?: string;
+}
+
+/**
+ * Parse YouTube URL into components
+ */
+function parseYouTubeUrl(url: string): YouTubeUrl | null {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname.replace(/^www\./, "");
+
+		// youtube.com/watch?v=VIDEO_ID
+		if ((hostname === "youtube.com" || hostname === "m.youtube.com") && parsed.pathname === "/watch") {
+			const videoId = parsed.searchParams.get("v");
+			const playlistId = parsed.searchParams.get("list") || undefined;
+			if (videoId) return { videoId, playlistId };
+		}
+
+		// youtube.com/v/VIDEO_ID or youtube.com/embed/VIDEO_ID
+		if (hostname === "youtube.com" || hostname === "m.youtube.com") {
+			const match = parsed.pathname.match(/^\/(v|embed)\/([a-zA-Z0-9_-]{11})/);
+			if (match) return { videoId: match[2] };
+		}
+
+		// youtu.be/VIDEO_ID
+		if (hostname === "youtu.be") {
+			const videoId = parsed.pathname.slice(1).split("/")[0];
+			if (videoId && /^[a-zA-Z0-9_-]{11}$/.test(videoId)) {
+				return { videoId };
+			}
+		}
+
+		// youtube.com/shorts/VIDEO_ID
+		if (hostname === "youtube.com" && parsed.pathname.startsWith("/shorts/")) {
+			const videoId = parsed.pathname.replace("/shorts/", "").split("/")[0];
+			if (videoId && /^[a-zA-Z0-9_-]{11}$/.test(videoId)) {
+				return { videoId };
+			}
+		}
+	} catch {}
+
+	return null;
+}
+
+/**
+ * Clean VTT subtitle content to plain text
+ */
+function cleanVttToText(vtt: string): string {
+	const lines = vtt.split("\n");
+	const textLines: string[] = [];
+	let lastLine = "";
+
+	for (const line of lines) {
+		// Skip WEBVTT header, timestamps, and metadata
+		if (
+			line.startsWith("WEBVTT") ||
+			line.startsWith("Kind:") ||
+			line.startsWith("Language:") ||
+			line.match(/^\d{2}:\d{2}/) || // Timestamp lines
+			line.match(/^[a-f0-9-]{36}$/) || // UUID cue identifiers
+			line.match(/^\d+$/) || // Numeric cue identifiers
+			line.includes("-->") ||
+			line.trim() === ""
+		) {
+			continue;
+		}
+
+		// Remove inline timestamp tags like <00:00:01.520>
+		let cleaned = line.replace(/<\d{2}:\d{2}:\d{2}\.\d{3}>/g, "");
+		// Remove other VTT tags like <c> </c>
+		cleaned = cleaned.replace(/<\/?[^>]+>/g, "");
+		cleaned = cleaned.trim();
+
+		// Skip duplicates (auto-generated captions often repeat)
+		if (cleaned && cleaned !== lastLine) {
+			textLines.push(cleaned);
+			lastLine = cleaned;
+		}
+	}
+
+	return textLines.join(" ").replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Format duration from seconds to human readable
+ */
+function formatDuration(seconds: number): string {
+	const h = Math.floor(seconds / 3600);
+	const m = Math.floor((seconds % 3600) / 60);
+	const s = Math.floor(seconds % 60);
+	if (h > 0) return `${h}:${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
+	return `${m}:${s.toString().padStart(2, "0")}`;
+}
+
+/**
+ * Handle YouTube URLs - fetch metadata and transcript
+ */
+export const handleYouTube: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	userSignal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	throwIfAborted(userSignal);
+	const yt = parseYouTubeUrl(url);
+	if (!yt) return null;
+
+	// Ensure yt-dlp is available (auto-download if missing)
+	const signal = ptree.combineSignals(userSignal, timeout * 1000);
+	const ytdlp = await ensureTool("yt-dlp", { signal, silent: true });
+	if (!ytdlp) {
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/plain",
+			method: "youtube-no-ytdlp",
+			content: "YouTube video detected but yt-dlp could not be installed.",
+			fetchedAt: new Date().toISOString(),
+			truncated: false,
+			notes: ["yt-dlp installation failed"],
+		};
+	}
+
+	const fetchedAt = new Date().toISOString();
+	const notes: string[] = [];
+	const videoUrl = `https://www.youtube.com/watch?v=${yt.videoId}`;
+	const execOptions = {
+		mode: "group" as const,
+		signal,
+		allowNonZero: true,
+		allowAbort: true,
+		stderr: "full" as const,
+	};
+
+	// Fetch video metadata
+	const metaResult = await ptree.exec(
+		[ytdlp, "--dump-json", "--no-warnings", "--no-playlist", "--skip-download", videoUrl],
+		execOptions,
+	);
+
+	let title = "YouTube Video";
+	let channel = "";
+	let description = "";
+	let duration = 0;
+	let uploadDate = "";
+	let viewCount = 0;
+
+	if (metaResult.ok && metaResult.stdout.trim()) {
+		try {
+			const meta = JSON.parse(metaResult.stdout) as {
+				title?: string;
+				channel?: string;
+				uploader?: string;
+				description?: string;
+				duration?: number;
+				upload_date?: string;
+				view_count?: number;
+			};
+			title = meta.title || title;
+			channel = meta.channel || meta.uploader || "";
+			description = meta.description || "";
+			duration = meta.duration || 0;
+			uploadDate = meta.upload_date || "";
+			viewCount = meta.view_count || 0;
+		} catch {}
+	}
+
+	// Format upload date
+	let formattedDate = "";
+	if (uploadDate && uploadDate.length === 8) {
+		formattedDate = `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}`;
+	}
+
+	// Try to fetch subtitles
+	let transcript = "";
+	let transcriptSource = "";
+
+	// First, list available subtitles
+	const listResult = await ptree.exec(
+		[ytdlp, "--list-subs", "--no-warnings", "--no-playlist", "--skip-download", videoUrl],
+		execOptions,
+	);
+
+	const hasManualSubs = listResult.stdout.includes("[info] Available subtitles");
+	const hasAutoSubs = listResult.stdout.includes("[info] Available automatic captions");
+
+	// Create temp directory for subtitle download
+	const tmpDir = os.tmpdir();
+	const tmpBase = path.join(tmpDir, `yt-${yt.videoId}-${Snowflake.next()}`);
+
+	try {
+		// Try manual subtitles first (English preferred)
+		if (hasManualSubs) {
+			const subResult = await ptree.exec(
+				[
+					ytdlp,
+					"--write-sub",
+					"--sub-lang",
+					"en,en-US,en-GB",
+					"--sub-format",
+					"vtt",
+					"--skip-download",
+					"--no-warnings",
+					"--no-playlist",
+					"-o",
+					tmpBase,
+					videoUrl,
+				],
+				execOptions,
+			);
+
+			if (subResult.ok) {
+				// Find the downloaded subtitle file using glob
+				const subFiles = await Array.fromAsync(new Bun.Glob(`${tmpBase}*.vtt`).scan({ absolute: true }));
+				if (subFiles.length > 0) {
+					const vttContent = await Bun.file(subFiles[0]).text();
+					transcript = cleanVttToText(vttContent);
+					transcriptSource = "manual";
+					notes.push("Using manual subtitles");
+				}
+			}
+		}
+
+		// Fall back to auto-generated captions
+		if (!transcript && hasAutoSubs) {
+			const autoResult = await ptree.exec(
+				[
+					ytdlp,
+					"--write-auto-sub",
+					"--sub-lang",
+					"en,en-US,en-GB",
+					"--sub-format",
+					"vtt",
+					"--skip-download",
+					"--no-warnings",
+					"--no-playlist",
+					"-o",
+					tmpBase,
+					videoUrl,
+				],
+				execOptions,
+			);
+
+			if (autoResult.ok) {
+				const subFiles = await Array.fromAsync(new Bun.Glob(`${tmpBase}*.vtt`).scan({ absolute: true }));
+				if (subFiles.length > 0) {
+					const vttContent = await Bun.file(subFiles[0]).text();
+					transcript = cleanVttToText(vttContent);
+					transcriptSource = "auto-generated";
+					notes.push("Using auto-generated captions");
+				}
+			}
+		}
+	} finally {
+		throwIfAborted(signal);
+		// Cleanup temp files (fire-and-forget with error suppression)
+		Array.fromAsync(new Bun.Glob(`${tmpBase}*`).scan({ absolute: true }))
+			.then(tmpFiles => Promise.all(tmpFiles.map(f => fs.unlink(f).catch(() => {}))))
+			.catch(() => {});
+	}
+
+	// Build markdown output
+	let md = `# ${title}\n\n`;
+	if (channel) md += `**Channel:** ${channel}\n`;
+	if (formattedDate) md += `**Uploaded:** ${formattedDate}\n`;
+	if (duration > 0) md += `**Duration:** ${formatDuration(duration)}\n`;
+	if (viewCount > 0) {
+		const formatted =
+			viewCount >= 1_000_000
+				? `${(viewCount / 1_000_000).toFixed(1)}M`
+				: viewCount >= 1_000
+					? `${(viewCount / 1_000).toFixed(1)}K`
+					: String(viewCount);
+		md += `**Views:** ${formatted}\n`;
+	}
+	md += `**Video ID:** ${yt.videoId}\n\n`;
+
+	if (description) {
+		// Truncate long descriptions
+		const descPreview = description.length > 1000 ? `${description.slice(0, 1000)}…` : description;
+		md += `---\n\n## Description\n\n${descPreview}\n\n`;
+	}
+
+	if (transcript) {
+		md += `---\n\n## Transcript (${transcriptSource})\n\n${transcript}\n`;
+	} else {
+		notes.push("No subtitles/captions available");
+		md += `---\n\n*No transcript available for this video.*\n`;
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: videoUrl,
+		contentType: "text/markdown",
+		method: "youtube",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes,
+	};
+};

package/src/web/search/auth.ts

@@ -0,0 +1,178 @@
+/**
+ * Anthropic Authentication
+ *
+ * 4-tier auth resolution:
+ *   1. ANTHROPIC_SEARCH_API_KEY / ANTHROPIC_SEARCH_BASE_URL env vars
+ *   2. Provider with api="anthropic-messages" in ~/.arcane/agent/models.json
+ *   3. OAuth credentials in ~/.arcane/agent/agent.db (with expiry check)
+ *   4. ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL fallback
+ */
+import { buildAnthropicHeaders as buildProviderAnthropicHeaders, getEnvApiKey } from "@nghyane/arcane-ai";
+import { $env, logger } from "@nghyane/arcane-utils";
+import { getAgentDbPath, getAgentDir } from "@nghyane/arcane-utils/dirs";
+import { AgentStorage } from "../../session/agent-storage";
+import type { AuthCredential } from "../../session/auth-storage";
+import type { AnthropicAuthConfig, AnthropicOAuthCredential, ModelsJson } from "./types";
+
+const DEFAULT_BASE_URL = "https://api.anthropic.com";
+
+/**
+ * Reads and parses a JSON file safely.
+ * @param filePath - Path to the JSON file
+ * @returns Parsed JSON content, or null if file doesn't exist or parsing fails
+ */
+async function readJson<T>(filePath: string): Promise<T | null> {
+	try {
+		const file = Bun.file(filePath);
+		if (!(await file.exists())) return null;
+		const content = await file.text();
+		return JSON.parse(content) as T;
+	} catch (error) {
+		logger.warn("Failed to parse JSON file", { path: filePath, error: String(error) });
+		return null;
+	}
+}
+
+/**
+ * Checks if a token is an OAuth token by looking for sk-ant-oat prefix.
+ * @param apiKey - The API key to check
+ * @returns True if the token is an OAuth token
+ */
+export function isOAuthToken(apiKey: string): boolean {
+	return apiKey.includes("sk-ant-oat");
+}
+
+/**
+ * Converts a generic AuthCredential to AnthropicOAuthCredential if it's a valid OAuth entry.
+ * @param credential - The credential to convert
+ * @returns The converted OAuth credential, or null if not a valid OAuth type
+ */
+function toAnthropicOAuthCredential(credential: AuthCredential): AnthropicOAuthCredential | null {
+	if (credential.type !== "oauth") return null;
+	if (typeof credential.access !== "string" || typeof credential.expires !== "number") return null;
+	return {
+		type: "oauth",
+		access: credential.access,
+		refresh: credential.refresh,
+		expires: credential.expires,
+	};
+}
+
+/**
+ * Reads Anthropic OAuth credentials from agent.db.
+ * @returns Array of valid Anthropic OAuth credentials
+ */
+async function readAnthropicOAuthCredentials(): Promise<AnthropicOAuthCredential[]> {
+	const storage = await AgentStorage.open(getAgentDbPath());
+	const records = storage.listAuthCredentials("anthropic");
+	const credentials: AnthropicOAuthCredential[] = [];
+	for (const record of records) {
+		const mapped = toAnthropicOAuthCredential(record.credential);
+		if (mapped) {
+			credentials.push(mapped);
+		}
+	}
+
+	return credentials;
+}
+
+/**
+ * Finds Anthropic auth config using 4-tier priority:
+ *   1. ANTHROPIC_SEARCH_API_KEY / ANTHROPIC_SEARCH_BASE_URL
+ *   2. Provider with api="anthropic-messages" in models.json
+ *   3. OAuth in agent.db (with 5-minute expiry buffer)
+ *   4. ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL fallback
+ * @returns The first valid auth configuration found, or null if none available
+ */
+export async function findAnthropicAuth(): Promise<AnthropicAuthConfig | null> {
+	const configDir = getAgentDir();
+
+	// 1. Explicit search-specific env vars
+	const searchApiKey = $env.ANTHROPIC_SEARCH_API_KEY;
+	const searchBaseUrl = $env.ANTHROPIC_SEARCH_BASE_URL;
+	if (searchApiKey) {
+		return {
+			apiKey: searchApiKey,
+			baseUrl: searchBaseUrl ?? DEFAULT_BASE_URL,
+			isOAuth: isOAuthToken(searchApiKey),
+		};
+	}
+
+	// 2. Provider with api="anthropic-messages" in models.json
+	const modelsJson = await readJson<ModelsJson>(`${configDir}/models.json`);
+	if (modelsJson?.providers) {
+		// First pass: look for providers with actual API keys
+		for (const [_name, provider] of Object.entries(modelsJson.providers)) {
+			if (provider.api === "anthropic-messages" && provider.apiKey && provider.apiKey !== "none") {
+				return {
+					apiKey: provider.apiKey,
+					baseUrl: provider.baseUrl ?? DEFAULT_BASE_URL,
+					isOAuth: isOAuthToken(provider.apiKey),
+				};
+			}
+		}
+		// Second pass: check for proxy mode (baseUrl but apiKey="none")
+		for (const [_name, provider] of Object.entries(modelsJson.providers)) {
+			if (provider.api === "anthropic-messages" && provider.baseUrl) {
+				return {
+					apiKey: provider.apiKey ?? "",
+					baseUrl: provider.baseUrl,
+					isOAuth: false,
+				};
+			}
+		}
+	}
+
+	// 3. OAuth credentials in agent.db (with 5-minute expiry buffer)
+	const expiryBuffer = 5 * 60 * 1000; // 5 minutes
+	const now = Date.now();
+	const credentials = await readAnthropicOAuthCredentials();
+	for (const credential of credentials) {
+		if (!credential.access) continue;
+		if (credential.expires > now + expiryBuffer) {
+			return {
+				apiKey: credential.access,
+				baseUrl: DEFAULT_BASE_URL,
+				isOAuth: true,
+			};
+		}
+	}
+
+	// 4. Generic ANTHROPIC_API_KEY fallback
+	const apiKey = getEnvApiKey("anthropic");
+	const baseUrl = $env.ANTHROPIC_BASE_URL;
+	if (apiKey) {
+		return {
+			apiKey,
+			baseUrl: baseUrl ?? DEFAULT_BASE_URL,
+			isOAuth: isOAuthToken(apiKey),
+		};
+	}
+
+	return null;
+}
+
+/**
+ * Builds HTTP headers for Anthropic API requests.
+ * @param auth - The authentication configuration
+ * @returns Headers object ready for use in fetch requests
+ */
+export function buildAnthropicHeaders(auth: AnthropicAuthConfig): Record<string, string> {
+	return buildProviderAnthropicHeaders({
+		apiKey: auth.apiKey,
+		baseUrl: auth.baseUrl,
+		isOAuth: auth.isOAuth,
+		extraBetas: ["web-search-2025-03-05"],
+		stream: false,
+	});
+}
+
+/**
+ * Builds the full API URL for Anthropic messages endpoint.
+ * @param auth - The authentication configuration
+ * @returns The complete API URL with beta query parameter
+ */
+export function buildAnthropicUrl(auth: AnthropicAuthConfig): string {
+	const base = `${auth.baseUrl}/v1/messages`;
+	return `${base}?beta=true`;
+}