@llmindset/hf-mcp 0.3.1 → 0.3.3

package/src/network/safe-fetch.ts

@@ -0,0 +1,174 @@
+import { assertExternalAddress } from './ip-policy.js';
+import { parseAndValidateUrl, type UrlPolicy } from './url-policy.js';
+
+export interface SafeFetchOptions {
+	urlPolicy: UrlPolicy;
+	timeoutMs?: number;
+	maxRedirects?: number;
+	externalOnly?: boolean;
+	requestInit?: RequestInit;
+	stripSensitiveHeadersOnCrossHostRedirect?: boolean;
+}
+
+export interface SafeFetchResult {
+	response: Response;
+	finalUrl: URL;
+	redirectsFollowed: number;
+}
+
+const DEFAULT_TIMEOUT_MS = 12500;
+const DEFAULT_MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const SENSITIVE_HEADERS = new Set(['authorization', 'proxy-authorization', 'cookie', 'x-hf-authorization']);
+
+function isRedirectStatus(status: number): boolean {
+	return REDIRECT_STATUSES.has(status);
+}
+
+function dropSensitiveHeaders(headersInit: HeadersInit | undefined): Headers {
+	const headers = new Headers(headersInit);
+	for (const key of SENSITIVE_HEADERS) {
+		headers.delete(key);
+	}
+	return headers;
+}
+
+function withMethodAndBody(requestInit: RequestInit, method: string, body: BodyInit | null | undefined): RequestInit {
+	const nextInit: RequestInit = {
+		...requestInit,
+		method,
+		redirect: 'manual',
+	};
+
+	if (body !== undefined && body !== null && method !== 'GET' && method !== 'HEAD') {
+		nextInit.body = body;
+	} else {
+		delete nextInit.body;
+	}
+
+	return nextInit;
+}
+
+async function fetchWithTimeout(url: URL, requestInit: RequestInit, timeoutMs: number): Promise<Response> {
+	if (timeoutMs <= 0) {
+		return fetch(url.toString(), {
+			...requestInit,
+			redirect: 'manual',
+		});
+	}
+
+	const outerSignal = requestInit.signal;
+
+	if (outerSignal) {
+		if (outerSignal.aborted) {
+			throw new Error('Request was aborted');
+		}
+	}
+
+	const timeoutSignal = AbortSignal.timeout(timeoutMs);
+	const signal = outerSignal ? AbortSignal.any([outerSignal, timeoutSignal]) : timeoutSignal;
+
+	try {
+		return await fetch(url.toString(), {
+			...requestInit,
+			signal,
+			redirect: 'manual',
+		});
+	} catch (error) {
+		if (error instanceof Error && error.name === 'AbortError') {
+			if (outerSignal?.aborted) {
+				throw new Error('Request was aborted');
+			}
+
+			if (timeoutSignal.aborted) {
+				throw new Error(`Request timed out after ${timeoutMs.toString()}ms`);
+			}
+
+			throw new Error(`Request timed out after ${timeoutMs.toString()}ms`);
+		}
+		throw error;
+	}
+}
+
+export async function safeFetch(url: string | URL, options: SafeFetchOptions): Promise<SafeFetchResult> {
+	const {
+		urlPolicy,
+		timeoutMs = DEFAULT_TIMEOUT_MS,
+		maxRedirects = DEFAULT_MAX_REDIRECTS,
+		externalOnly = false,
+		requestInit = {},
+		stripSensitiveHeadersOnCrossHostRedirect = true,
+	} = options;
+
+	if (maxRedirects < 0) {
+		throw new Error('maxRedirects must be >= 0');
+	}
+
+	let currentUrl = parseAndValidateUrl(url, urlPolicy);
+	if (externalOnly) {
+		await assertExternalAddress(currentUrl.hostname);
+	}
+
+	const baseHeaders = new Headers(requestInit.headers);
+	let currentMethod = (requestInit.method || 'GET').toUpperCase();
+	let currentBody = requestInit.body;
+	let redirectsFollowed = 0;
+
+	while (true) {
+		const currentInit = withMethodAndBody(
+			{
+				...requestInit,
+				headers: baseHeaders,
+			},
+			currentMethod,
+			currentBody
+		);
+
+		const response = await fetchWithTimeout(currentUrl, currentInit, timeoutMs);
+
+		if (!isRedirectStatus(response.status)) {
+			return {
+				response,
+				finalUrl: currentUrl,
+				redirectsFollowed,
+			};
+		}
+
+		if (redirectsFollowed >= maxRedirects) {
+			throw new Error(`Redirect limit exceeded (${maxRedirects.toString()})`);
+		}
+
+		const location = response.headers.get('location');
+		if (!location) {
+			throw new Error(`Redirect response missing Location header (status ${response.status.toString()})`);
+		}
+
+		const nextCandidate = new URL(location, currentUrl);
+		const nextUrl = parseAndValidateUrl(nextCandidate, urlPolicy);
+		if (externalOnly) {
+			await assertExternalAddress(nextUrl.hostname);
+		}
+
+		if (stripSensitiveHeadersOnCrossHostRedirect && currentUrl.origin !== nextUrl.origin) {
+			const filtered = dropSensitiveHeaders(baseHeaders);
+			baseHeaders.forEach((_, key) => {
+				baseHeaders.delete(key);
+			});
+			filtered.forEach((value, key) => {
+				baseHeaders.set(key, value);
+			});
+		}
+
+		if (response.status === 303 || ((response.status === 301 || response.status === 302) && currentMethod === 'POST')) {
+			currentMethod = 'GET';
+			currentBody = undefined;
+			baseHeaders.delete('content-length');
+			baseHeaders.delete('content-type');
+		}
+
+		redirectsFollowed += 1;
+		currentUrl = nextUrl;
+
+		await response.body?.cancel();
+	}
+}

package/src/network/url-policy.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, it } from 'vitest';
+import {
+	createExactHostPolicy,
+	createExternalHttpsPolicy,
+	createGradioMcpHostPolicy,
+	createGradioSchemaHostPolicy,
+	createHuggingFaceHubPolicy,
+	createLocalhostHttpPolicy,
+	createGradioMcpPolicy,
+	createHfDocsPolicy,
+	isLocalhostHostname,
+	parseAndValidateUrl,
+} from './url-policy.js';
+
+describe('url-policy', () => {
+	describe('HF docs policy', () => {
+		it('accepts valid Hugging Face docs and Gradio docs URLs', () => {
+			expect(() => parseAndValidateUrl('https://huggingface.co/docs/transformers', createHfDocsPolicy())).not.toThrow();
+			expect(() => parseAndValidateUrl('https://www.huggingface.co/docs/datasets', createHfDocsPolicy())).not.toThrow();
+			expect(() => parseAndValidateUrl('https://www.gradio.app/guides', createHfDocsPolicy())).not.toThrow();
+		});
+
+		it('rejects non-https or non-allowlisted hosts', () => {
+			expect(() => parseAndValidateUrl('http://huggingface.co/docs/transformers', createHfDocsPolicy())).toThrow(
+				'URL protocol is not allowed'
+			);
+			expect(() => parseAndValidateUrl('https://example.com/docs/transformers', createHfDocsPolicy())).toThrow(
+				'URL hostname is not allowed'
+			);
+		});
+
+		it('rejects traversal and encoded traversal variants', () => {
+			const variants = [
+				'https://huggingface.co/docs/../x',
+				'https://huggingface.co/docs/%2e%2e/x',
+				'https://huggingface.co/docs/%2e%2e%2fx',
+				'https://huggingface.co/docs/..%2fx',
+				'https://huggingface.co/docs/%2e%2e%5cx',
+				'https://huggingface.co/docs/%252e%252e%252fx',
+			];
+
+			for (const candidate of variants) {
+				expect(() => parseAndValidateUrl(candidate, createHfDocsPolicy())).toThrow();
+			}
+		});
+
+		it('enforces /docs/ prefix on HF hosts', () => {
+			expect(() => parseAndValidateUrl('https://huggingface.co/models/some-model', createHfDocsPolicy())).toThrow(
+				'Hugging Face docs URLs must remain under /docs/'
+			);
+		});
+	});
+
+	describe('Gradio MCP policy', () => {
+		it('accepts mcp endpoint URLs over https', () => {
+			expect(() =>
+				parseAndValidateUrl('https://demo-space.hf.space/gradio_api/mcp/', createGradioMcpPolicy())
+			).not.toThrow();
+			expect(() =>
+				parseAndValidateUrl('https://fake-mcp.local/gradio_api/mcp/', createGradioMcpPolicy())
+			).not.toThrow();
+		});
+
+		it('rejects invalid paths', () => {
+			expect(() => parseAndValidateUrl('https://demo-space.hf.space/not-mcp', createGradioMcpPolicy())).toThrow(
+				'URL path must start with /gradio_api/mcp'
+			);
+		});
+	});
+
+	describe('external https policy', () => {
+		it('rejects credentials in URL', () => {
+			expect(() => parseAndValidateUrl('https://user:pass@example.com/file.wav', createExternalHttpsPolicy())).toThrow(
+				'URL credentials are not allowed'
+			);
+		});
+	});
+
+	it('supports huggingface hub and localhost policy helpers', () => {
+		expect(() => parseAndValidateUrl('https://huggingface.co/api/models', createHuggingFaceHubPolicy())).not.toThrow();
+		expect(() => parseAndValidateUrl('http://localhost:7860/health', createLocalhostHttpPolicy())).not.toThrow();
+		expect(() => parseAndValidateUrl('https://example.com/x', createExactHostPolicy('example.com', 'https:'))).not.toThrow();
+		expect(() =>
+			parseAndValidateUrl(
+				'https://demo-space.hf.space/gradio_api/mcp/schema',
+				createGradioSchemaHostPolicy('demo-space.hf.space')
+			)
+		).not.toThrow();
+		expect(() =>
+			parseAndValidateUrl(
+				'https://demo-space.hf.space/gradio_api/mcp/',
+				createGradioMcpHostPolicy('demo-space.hf.space', 'https:')
+			)
+		).not.toThrow();
+		expect(isLocalhostHostname('localhost')).toBe(true);
+		expect(isLocalhostHostname('127.0.0.1')).toBe(true);
+		expect(isLocalhostHostname('[::1]')).toBe(true);
+		expect(isLocalhostHostname('example.com')).toBe(false);
+	});
+});

package/src/network/url-policy.ts

@@ -0,0 +1,304 @@
+export interface UrlPathRules {
+	requiredPrefix?: string;
+}
+
+export type UrlProtocol = 'https:' | 'http:';
+
+export interface UrlQueryRules {
+	allowAny?: boolean;
+	allowKeys?: ReadonlySet<string>;
+}
+
+export interface UrlPolicy {
+	allowedProtocols: ReadonlySet<UrlProtocol>;
+	allowedHosts?: ReadonlySet<string>;
+	allowSubdomainsOf?: readonly string[];
+	requireDefaultPort?: boolean;
+	pathRules?: UrlPathRules;
+	queryRules?: UrlQueryRules;
+	allowCredentials?: boolean;
+	customValidator?: (url: URL) => void;
+}
+
+const LOCALHOST_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', '::1']);
+
+const ENCODED_SEPARATOR_RE = /%(?:2f|5c)/i;
+const ENCODED_BYTE_RE = /%[0-9a-f]{2}/i;
+const INVALID_PERCENT_ENCODING_RE = /%(?![0-9a-f]{2})/i;
+
+function normalizeHostname(hostname: string): string {
+	return hostname.toLowerCase().replace(/\.+$/, '');
+}
+
+function safeDecodeURIComponent(value: string): string {
+	try {
+		return decodeURIComponent(value);
+	} catch {
+		throw new Error('URL contains invalid percent-encoding');
+	}
+}
+
+function collectDecodedPathVariants(pathname: string): string[] {
+	const variants = [pathname];
+	let current = pathname;
+
+	for (let i = 0; i < 2; i += 1) {
+		if (!current.includes('%')) {
+			break;
+		}
+
+		const decoded = safeDecodeURIComponent(current);
+		if (decoded === current) {
+			break;
+		}
+
+		variants.push(decoded);
+		current = decoded;
+	}
+
+	return variants;
+}
+
+function hasDotSegments(pathname: string): boolean {
+	const normalized = pathname.replace(/\\/g, '/');
+	const segments = normalized.split('/');
+	return segments.some((segment) => segment === '.' || segment === '..');
+}
+
+function matchesRequiredPrefix(pathname: string, requiredPrefix: string): boolean {
+	const normalizedPath = pathname.replace(/\\/g, '/');
+	const normalizedPrefix = requiredPrefix.replace(/\\/g, '/');
+
+	if (normalizedPath === normalizedPrefix) {
+		return true;
+	}
+
+	if (normalizedPrefix.endsWith('/') && normalizedPath === normalizedPrefix.slice(0, -1)) {
+		return true;
+	}
+
+	return normalizedPath.startsWith(normalizedPrefix);
+}
+
+function assertHostAllowed(hostname: string, policy: UrlPolicy): void {
+	if (!policy.allowedHosts && (!policy.allowSubdomainsOf || policy.allowSubdomainsOf.length === 0)) {
+		return;
+	}
+
+	const normalized = normalizeHostname(hostname);
+
+	if (policy.allowedHosts) {
+		for (const host of policy.allowedHosts) {
+			if (normalizeHostname(host) === normalized) {
+				return;
+			}
+		}
+	}
+
+	if (policy.allowSubdomainsOf) {
+		for (const domain of policy.allowSubdomainsOf) {
+			const normalizedDomain = normalizeHostname(domain);
+			if (normalized === normalizedDomain || normalized.endsWith(`.${normalizedDomain}`)) {
+				return;
+			}
+		}
+	}
+
+	throw new Error(`URL hostname is not allowed: ${hostname}`);
+}
+
+function assertPathAllowed(url: URL, pathRules?: UrlPathRules): void {
+	const pathname = url.pathname;
+
+	if (pathname.includes('%') && INVALID_PERCENT_ENCODING_RE.test(pathname)) {
+		throw new Error('URL path contains invalid percent-encoding');
+	}
+
+	const variants = collectDecodedPathVariants(pathname);
+
+	const hasEncodedSeparators = variants.some((variant) => ENCODED_SEPARATOR_RE.test(variant));
+	if (hasEncodedSeparators) {
+		throw new Error('URL path contains encoded separators');
+	}
+
+	const hasUnsafeDotSegments = variants.some((variant) => hasDotSegments(variant));
+	if (hasUnsafeDotSegments) {
+		throw new Error('URL path contains dot-segments');
+	}
+
+	if (variants.length > 1) {
+		const decodedOnce = variants[1] ?? '';
+		if (ENCODED_BYTE_RE.test(decodedOnce)) {
+			throw new Error('URL path appears to use double-encoding');
+		}
+	}
+
+	if (pathRules?.requiredPrefix) {
+		const hasPrefix = variants.some((variant) => matchesRequiredPrefix(variant, pathRules.requiredPrefix ?? ''));
+		if (!hasPrefix) {
+			throw new Error(`URL path must start with ${pathRules.requiredPrefix}`);
+		}
+	}
+}
+
+function assertQueryAllowed(url: URL, policy: UrlPolicy): void {
+	const rules = policy.queryRules;
+	if (!rules || rules.allowAny === true) {
+		return;
+	}
+
+	if (!rules.allowKeys) {
+		if (url.search.length > 0) {
+			throw new Error('URL query string is not allowed');
+		}
+		return;
+	}
+
+	for (const key of url.searchParams.keys()) {
+		if (!rules.allowKeys.has(key)) {
+			throw new Error(`URL query parameter is not allowed: ${key}`);
+		}
+	}
+}
+
+function assertPortAllowed(url: URL, policy: UrlPolicy): void {
+	if (!policy.requireDefaultPort || url.port.length === 0) {
+		return;
+	}
+
+	const defaultPort = url.protocol === 'https:' ? '443' : url.protocol === 'http:' ? '80' : '';
+	if (!defaultPort || url.port !== defaultPort) {
+		throw new Error(`URL port is not allowed for protocol ${url.protocol}`);
+	}
+}
+
+export function validateUrlAgainstPolicy(url: URL, policy: UrlPolicy): void {
+	if (!policy.allowedProtocols.has(url.protocol as UrlProtocol)) {
+		throw new Error(`URL protocol is not allowed: ${url.protocol}`);
+	}
+
+	if (!policy.allowCredentials && (url.username.length > 0 || url.password.length > 0)) {
+		throw new Error('URL credentials are not allowed');
+	}
+
+	assertHostAllowed(url.hostname, policy);
+	assertPortAllowed(url, policy);
+	assertPathAllowed(url, policy.pathRules);
+	assertQueryAllowed(url, policy);
+
+	policy.customValidator?.(url);
+}
+
+export function parseAndValidateUrl(input: string | URL, policy: UrlPolicy): URL {
+	const parsed = input instanceof URL ? new URL(input.toString()) : new URL(input.trim());
+	validateUrlAgainstPolicy(parsed, policy);
+	return parsed;
+}
+
+export function createHfDocsPolicy(): UrlPolicy {
+	const hfHosts = new Set(['huggingface.co', 'www.huggingface.co']);
+
+	return {
+		allowedProtocols: new Set(['https:']),
+		allowedHosts: new Set(['huggingface.co', 'www.huggingface.co', 'gradio.app', 'www.gradio.app']),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+		customValidator: (url) => {
+			const host = normalizeHostname(url.hostname);
+			if (hfHosts.has(host) && !matchesRequiredPrefix(url.pathname, '/docs/')) {
+				throw new Error('Hugging Face docs URLs must remain under /docs/');
+			}
+		},
+	};
+}
+
+export function createGradioMcpPolicy(): UrlPolicy {
+	return {
+		allowedProtocols: new Set(['https:', 'http:']),
+		pathRules: {
+			requiredPrefix: '/gradio_api/mcp',
+		},
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+		customValidator: (url) => {
+			const enforceLocalHttpOnly = process.env.NODE_ENV === 'production';
+			if (enforceLocalHttpOnly && url.protocol === 'http:' && !isLocalhostHostname(url.hostname)) {
+				throw new Error('HTTP is only allowed for localhost Gradio MCP endpoints');
+			}
+		},
+	};
+}
+
+export function isLocalhostHostname(hostname: string): boolean {
+	return LOCALHOST_HOSTS.has(normalizeHostname(hostname));
+}
+
+export function createExternalHttpsPolicy(): UrlPolicy {
+	return {
+		allowedProtocols: new Set(['https:']),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}
+
+export function createHuggingFaceHubPolicy(): UrlPolicy {
+	return {
+		allowedProtocols: new Set(['https:']),
+		allowedHosts: new Set(['huggingface.co', 'www.huggingface.co', 'hf.co']),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}
+
+export function createLocalhostHttpPolicy(): UrlPolicy {
+	return {
+		allowedProtocols: new Set(['https:', 'http:']),
+		allowedHosts: new Set(['localhost', '127.0.0.1', '[::1]']),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}
+
+export function createExactHostPolicy(hostname: string, allowedProtocol: UrlProtocol): UrlPolicy {
+	return {
+		allowedProtocols: new Set([allowedProtocol]),
+		allowedHosts: new Set([hostname.toLowerCase()]),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}
+
+export function createHostPrefixPolicy(
+	hostname: string,
+	requiredPrefix: string,
+	allowedProtocol: UrlProtocol = 'https:'
+): UrlPolicy {
+	return {
+		allowedProtocols: new Set([allowedProtocol]),
+		allowedHosts: new Set([hostname.toLowerCase()]),
+		pathRules: {
+			requiredPrefix,
+		},
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}
+
+export function createGradioMcpHostPolicy(
+	hostname: string,
+	allowedProtocol: UrlProtocol
+): UrlPolicy {
+	return createHostPrefixPolicy(hostname, '/gradio_api/mcp', allowedProtocol);
+}
+
+export function createGradioSchemaHostPolicy(hostname: string): UrlPolicy {
+	return createHostPrefixPolicy(hostname, '/gradio_api/mcp/schema');
+}
+
+export function createHttpOrHttpsPolicy(): UrlPolicy {
+	return {
+		allowedProtocols: new Set(['https:', 'http:']),
+		allowCredentials: false,
+		queryRules: { allowAny: true },
+	};
+}

package/src/readme-utils.ts

@@ -2,12 +2,14 @@
  * Utility functions for fetching and processing README files from Hugging Face repositories
  */
 
+import { fetchWithProfile, NETWORK_FETCH_PROFILES } from './network/fetch-profile.js';
+
 // Maximum number of characters to include from a README
 const DEFAULT_MAX_README_CHARS = 10_000;
 
 /**
  * Fetches README content from a Hugging Face repository
- * 
+ *
  * @param repoName The resolved repository name (e.g., 'rajpurkar/squad', 'openai-community/gpt2')
  * @param type The repository type ('models' or 'datasets')
  * @param includeYaml Whether to include YAML frontmatter (default: false)
@@ -20,14 +22,13 @@ export async function fetchReadmeContent(
 ): Promise<string | null> {
 	try {
 		// Construct the URL based on repository type
-		const baseUrl = type === 'datasets' 
-			? `https://huggingface.co/datasets/${repoName}`
-			: `https://huggingface.co/${repoName}`;
-		
+		const baseUrl =
+			type === 'datasets' ? `https://huggingface.co/datasets/${repoName}` : `https://huggingface.co/${repoName}`;
+
 		const url = `${baseUrl}/resolve/main/README.md`;
 
-		const response = await fetch(url);
-		
+		const { response } = await fetchWithProfile(url, NETWORK_FETCH_PROFILES.hfHub());
+
 		if (!response.ok) {
 			if (response.status === 404) {
 				// README doesn't exist, return null silently
@@ -64,7 +65,7 @@ export async function fetchReadmeContent(
 
 /**
  * Strips YAML frontmatter from markdown content
- * 
+ *
  * @param content The full markdown content
  * @returns The content with YAML frontmatter removed
  */
@@ -72,12 +73,12 @@ function stripYamlFrontmatter(content: string): string {
 	// Match YAML frontmatter: starts with ---, ends with ---
 	const yamlPattern = /^(\s*---[\r\n]+)([\S\s]*?)([\r\n]+---(\r\n|\n|$))/;
 	const match = content.match(yamlPattern);
-	
+
 	if (match) {
 		// Return everything after the closing ---
 		return content.substring(match[0].length);
 	}
-	
+
 	// No YAML frontmatter found, return original content
 	return content;
 }