@llmindset/hf-mcp 0.1.16 → 0.1.18

package/src/docs-search/docs-semantic-search.test.ts

@@ -0,0 +1,318 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { DocSearchTool } from './docs-semantic-search.js';
+
+// Mock the fetch function
+const mockFetch = vi.fn();
+global.fetch = mockFetch as typeof fetch;
+
+describe('DocSearchTool', () => {
+	let docSearchTool: DocSearchTool;
+
+	beforeEach(() => {
+		docSearchTool = new DocSearchTool();
+		vi.clearAllMocks();
+	});
+
+	describe('search', () => {
+		it('should return "No query provided" when query is empty', async () => {
+			const result = await docSearchTool.search('');
+			expect(result).toBe('No query provided');
+		});
+
+		it('should return no results message when API returns empty array', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve([]),
+			});
+
+			const result = await docSearchTool.search('nonexistent');
+			expect(result).toBe(`No documentation found for query 'nonexistent'`);
+		});
+
+		it('should return no results message with product filter', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve([]),
+			});
+
+			const result = await docSearchTool.search('nonexistent', 'hub');
+			expect(result).toBe(`No documentation found for query 'nonexistent' in product 'hub'`);
+		});
+
+		it('should format results grouped by product and page', async () => {
+			const sampleResults = [
+				{
+					text: 'Download a comprehensive CSV file containing analytics',
+					product: 'hub',
+					heading1: 'Analytics',
+					source_page_url: 'https://huggingface.co/docs/hub/enterprise-hub-analytics#export-analytics-as-csv',
+					source_page_title: 'Enterprise-hub-analytics',
+					heading2: 'Export Analytics as CSV',
+				},
+				{
+					text: 'View analytics for your repositories',
+					product: 'hub',
+					heading1: 'Analytics',
+					source_page_url: 'https://huggingface.co/docs/hub/enterprise-hub-analytics#export-analytics-as-csv',
+					source_page_title: 'Enterprise-hub-analytics',
+					heading2: 'View Analytics',
+				},
+				{
+					text: 'In this quickstart, you will learn how to use the dataset viewer REST API',
+					product: 'dataset-viewer',
+					heading1: 'Quickstart',
+					source_page_url: 'https://huggingface.co/docs/dataset-viewer/quick_start#quickstart',
+					source_page_title: 'Quick start',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('analytics');
+
+			// Check header
+			expect(result).toContain('# Documentation Search Results for "analytics"');
+			expect(result).toContain('Found 3 results');
+
+			// Check product grouping - hub should come before dataset-viewer (hub has 2 results, dataset-viewer has 1)
+			const hubIndex = result.indexOf('## Results for Product: hub');
+			const datasetViewerIndex = result.indexOf('## Results for Product: dataset-viewer');
+			expect(hubIndex).toBeLessThan(datasetViewerIndex);
+			expect(hubIndex).toBeGreaterThan(-1);
+			expect(datasetViewerIndex).toBeGreaterThan(-1);
+
+			// Check that result counts are shown
+			expect(result).toContain('## Results for Product: hub (2 results)');
+			expect(result).toContain('## Results for Product: dataset-viewer (1 results)');
+
+			// Check page links
+			expect(result).toContain('### Results from [Analytics](https://huggingface.co/docs/hub/enterprise-hub-analytics#export-analytics-as-csv)');
+			expect(result).toContain('### Results from [Quickstart](https://huggingface.co/docs/dataset-viewer/quick_start#quickstart)');
+
+			// Check excerpts with heading2
+			expect(result).toContain('**Excerpt from "Export Analytics as CSV":**');
+			expect(result).toContain('**Excerpt from "View Analytics":**');
+
+			// Check excerpt content appears as plain text
+			expect(result).toContain('Download a comprehensive CSV file containing analytics');
+			expect(result).toContain('View analytics for your repositories');
+			expect(result).toContain('In this quickstart, you will learn how to use the dataset viewer REST API');
+
+			// Check footer
+			expect(result).toContain('Use the "fetch_hf_doc" tool to download a specific document.');
+		});
+
+		it('should handle results without heading2', async () => {
+			const sampleResults = [
+				{
+					text: 'This is a simple text without heading2',
+					product: 'transformers',
+					heading1: 'Introduction',
+					source_page_url: 'https://huggingface.co/docs/transformers/index',
+					source_page_title: 'Transformers',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('transformers');
+
+			// Should not contain "Excerpt from" when heading2 is missing
+			expect(result).not.toContain('**Excerpt from');
+			expect(result).toContain('This is a simple text without heading2');
+		});
+
+		it('should properly escape markdown special characters', async () => {
+			const sampleResults = [
+				{
+					text: 'Text with [brackets] and *asterisks* and _underscores_',
+					product: 'hub',
+					heading1: 'Special * Characters',
+					source_page_url: 'https://huggingface.co/docs/hub/test',
+					source_page_title: 'Test',
+					heading2: 'Section with [brackets]',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('special');
+
+			// Check that special characters are escaped in headings and page titles
+			expect(result).toContain('Special \\* Characters');
+			// Note: heading2 appears in bold text, which doesn't get escaped
+			expect(result).toContain('"Section with [brackets]"');
+		});
+
+		it('should clean HTML tags from text', async () => {
+			const sampleResults = [
+				{
+					text: 'Text with <div class="test">HTML tags</div> and <img src="test.png" alt="image"/>',
+					product: 'hub',
+					heading1: 'HTML Test',
+					source_page_url: 'https://huggingface.co/docs/hub/html-test',
+					source_page_title: 'HTML Test',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('html');
+
+			// HTML tags should be removed
+			expect(result).toContain('Text with HTML tags and');
+			expect(result).not.toContain('<div');
+			expect(result).not.toContain('<img');
+		});
+
+		it('should sort multiple products and pages correctly by count', async () => {
+			const sampleResults = [
+				// Hub has 3 results (should be first)
+				{
+					text: 'Result from hub page 1 - first',
+					product: 'hub',
+					heading1: 'Page 1',
+					source_page_url: 'https://huggingface.co/docs/hub/page1',
+					source_page_title: 'Page 1',
+				},
+				{
+					text: 'Result from hub page 1 - second',
+					product: 'hub',
+					heading1: 'Page 1',
+					source_page_url: 'https://huggingface.co/docs/hub/page1',
+					source_page_title: 'Page 1',
+				},
+				{
+					text: 'Result from hub page 2',
+					product: 'hub',
+					heading1: 'Page 2',
+					source_page_url: 'https://huggingface.co/docs/hub/page2',
+					source_page_title: 'Page 2',
+				},
+				// Transformers has 1 result (should be second)
+				{
+					text: 'Result from transformers',
+					product: 'transformers',
+					heading1: 'Transformers Page',
+					source_page_url: 'https://huggingface.co/docs/transformers/page1',
+					source_page_title: 'Transformers',
+				},
+				// Datasets has 1 result (should be third)
+				{
+					text: 'Result from datasets',
+					product: 'datasets',
+					heading1: 'Datasets Page',
+					source_page_url: 'https://huggingface.co/docs/datasets/page1',
+					source_page_title: 'Datasets',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('test');
+
+			// Check product order by count: hub (3) > transformers (1) = datasets (1) 
+			const hubIndex = result.indexOf('## Results for Product: hub');
+			const transformersIndex = result.indexOf('## Results for Product: transformers');
+			const datasetsIndex = result.indexOf('## Results for Product: datasets');
+
+			expect(hubIndex).toBeLessThan(transformersIndex);
+			expect(hubIndex).toBeLessThan(datasetsIndex);
+
+			// Check that hub shows total count
+			expect(result).toContain('## Results for Product: hub (3 results)');
+			
+			// Check page order within hub product: page1 (2 results) should come before page2 (1 result)
+			const page1Index = result.indexOf('https://huggingface.co/docs/hub/page1');
+			const page2Index = result.indexOf('https://huggingface.co/docs/hub/page2');
+			expect(page1Index).toBeLessThan(page2Index);
+
+			// Check that page1 shows its multiple results count
+			expect(result).toContain('### Results from [Page 1](https://huggingface.co/docs/hub/page1) (2 results)');
+		});
+
+		it('should include product filter in API call when provided', async () => {
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve([]),
+			});
+
+			await docSearchTool.search('test', 'hub');
+
+			expect(mockFetch).toHaveBeenCalledWith(
+				expect.stringContaining('q=test&product=hub'),
+				expect.any(Object)
+			);
+		});
+
+		it('should handle API errors gracefully', async () => {
+			mockFetch.mockRejectedValueOnce(new Error('Network error'));
+
+			await expect(docSearchTool.search('test')).rejects.toThrow('Failed to search documentation:');
+		});
+	});
+
+	describe('groupResults', () => {
+		it('should group results by product and page URL', async () => {
+			const sampleResults = [
+				{
+					text: 'Result 1',
+					product: 'hub',
+					heading1: 'Page 1',
+					source_page_url: 'https://example.com/page1',
+					source_page_title: 'Page 1',
+				},
+				{
+					text: 'Result 2',
+					product: 'hub',
+					heading1: 'Page 1',
+					source_page_url: 'https://example.com/page1',
+					source_page_title: 'Page 1',
+				},
+				{
+					text: 'Result 3',
+					product: 'transformers',
+					heading1: 'Page 2',
+					source_page_url: 'https://example.com/page2',
+					source_page_title: 'Page 2',
+				},
+			];
+
+			mockFetch.mockResolvedValueOnce({
+				ok: true,
+				json: () => Promise.resolve(sampleResults),
+			});
+
+			const result = await docSearchTool.search('test');
+
+			// Verify grouping structure in output
+			expect(result).toContain('## Results for Product: hub');
+			expect(result).toContain('## Results for Product: transformers');
+			
+			// Verify that both results from the same page are together
+			const result1Index = result.indexOf('Result 1');
+			const result2Index = result.indexOf('Result 2');
+			const result3Index = result.indexOf('Result 3');
+			
+			// Results 1 and 2 should be close together (same page)
+			expect(Math.abs(result2Index - result1Index)).toBeLessThan(100);
+			// Result 3 should be further away (different product)
+			expect(Math.abs(result3Index - result1Index)).toBeGreaterThan(50);
+		});
+	});
+});

package/src/docs-search/docs-semantic-search.ts

@@ -0,0 +1,207 @@
+import { z } from 'zod';
+import { HfApiCall } from '../hf-api-call.js';
+import { escapeMarkdown } from '../utilities.js';
+import { DOC_FETCH_CONFIG } from './doc-fetch.js';
+
+export const DOCS_SEMANTIC_SEARCH_CONFIG = {
+	name: 'docs_search',
+	description:
+		'Search the Hugging Face documentation library with semantic search. Returns documentation excerpts ' +
+		'grouped by Product and document page.',
+	schema: z.object({
+		query: z
+			.string()
+			.min(3, 'Supply at least one search term')
+			.max(200, 'Query too long')
+			.describe('Semantic search query'),
+		product: z
+			.string()
+			.optional()
+			.describe('Filter by specific product (e.g., "hub", "dataset-viewer", "transformers")'),
+	}),
+	annotations: {
+		title: 'Hugging Face Documentation Search',
+		destructiveHint: false,
+		readOnlyHint: true,
+		openWorldHint: true,
+	},
+} as const;
+
+export type DocSearchParams = z.infer<typeof DOCS_SEMANTIC_SEARCH_CONFIG.schema>;
+
+interface DocSearchResult {
+	text: string;
+	product: string;
+	heading1: string;
+	source_page_url: string;
+	source_page_title: string;
+	heading2?: string;
+}
+
+interface DocSearchApiParams {
+	q: string;
+	product?: string;
+}
+
+/**
+ * Use the Hugging Face Semantic Document Search API
+ */
+export class DocSearchTool extends HfApiCall<DocSearchApiParams, DocSearchResult[]> {
+	/**
+	 * @param apiUrl The URL of the Hugging Face document search API
+	 * @param hfToken Optional Hugging Face token for API access
+	 */
+	constructor(hfToken?: string, apiUrl = 'https://hf.co/api/docs/search') {
+		super(apiUrl, hfToken);
+	}
+
+	/**
+	 * @param query Search query string (e.g. "rate limits", "analytics")
+	 * @param product Optional product filter
+	 */
+	async search(query: string, product?: string): Promise<string> {
+		try {
+			if (!query) return 'No query provided';
+
+			const params: DocSearchApiParams = { q: query.toLowerCase() };
+			if (product) {
+				params.product = product;
+			}
+
+			const results = await this.callApi<DocSearchResult[]>(params);
+
+			if (results.length === 0) {
+				return product
+					? `No documentation found for query '${query}' in product '${product}'`
+					: `No documentation found for query '${query}'`;
+			}
+
+			return formatSearchResults(query, results, product);
+		} catch (error) {
+			if (error instanceof Error) {
+				throw new Error(`Failed to search documentation: ${error.message}`);
+			}
+			throw error;
+		}
+	}
+}
+
+/**
+ * Group results by product and source page URL
+ */
+function groupResults(results: DocSearchResult[]): Map<string, Map<string, DocSearchResult[]>> {
+	const grouped = new Map<string, Map<string, DocSearchResult[]>>();
+
+	for (const result of results) {
+		if (!grouped.has(result.product)) {
+			grouped.set(result.product, new Map());
+		}
+
+		const productGroup = grouped.get(result.product);
+		if (!productGroup) continue;
+
+		if (!productGroup.has(result.source_page_url)) {
+			productGroup.set(result.source_page_url, []);
+		}
+
+		const pageResults = productGroup.get(result.source_page_url);
+		if (pageResults) {
+			pageResults.push(result);
+		}
+	}
+
+	return grouped;
+}
+
+/**
+ * Format a single result excerpt
+ */
+function formatExcerpt(result: DocSearchResult): string {
+	const lines: string[] = [];
+
+	if (result.heading2) {
+		lines.push(`**Excerpt from "${escapeMarkdown(result.heading2)}":**`);
+	}
+
+	// Clean up the text - remove HTML tags if any
+	const cleanText = result.text
+		.replace(/<[^>]*>/g, '')
+		.replace(/\n\s*\n/g, '\n')
+		.trim();
+
+	lines.push(cleanText);
+	lines.push('');
+
+	return lines.join('\n');
+}
+
+/**
+ * Format search results grouped by product and page
+ */
+function formatSearchResults(query: string, results: DocSearchResult[], productFilter?: string): string {
+	const lines: string[] = [];
+
+	// Header
+	const filterText = productFilter ? ` (filtered by product: ${productFilter})` : '';
+	lines.push(`# Documentation Search Results for "${escapeMarkdown(query)}"${filterText}`);
+	lines.push('');
+	lines.push(`Found ${results.length} results`);
+	lines.push('');
+
+	// Group results
+	const grouped = groupResults(results);
+
+	// Sort products by count (most hits first)
+	const sortedProducts = Array.from(grouped.keys()).sort((a, b) => {
+		const productGroupA = grouped.get(a);
+		const productGroupB = grouped.get(b);
+		if (!productGroupA || !productGroupB) return 0;
+
+		const countA = Array.from(productGroupA.values()).reduce((sum, arr) => sum + arr.length, 0);
+		const countB = Array.from(productGroupB.values()).reduce((sum, arr) => sum + arr.length, 0);
+		return countB - countA; // Descending order
+	});
+
+	for (const product of sortedProducts) {
+		const productGroup = grouped.get(product);
+		if (!productGroup) continue;
+
+		const totalProductHits = Array.from(productGroup.values()).reduce((sum, arr) => sum + arr.length, 0);
+		lines.push(`## Results for Product: ${escapeMarkdown(product)} (${totalProductHits} results)`);
+		lines.push('');
+
+		// Sort URLs within each product by count (most hits first)
+		const sortedUrls = Array.from(productGroup.keys()).sort((a, b) => {
+			const pageResultsA = productGroup.get(a);
+			const pageResultsB = productGroup.get(b);
+			if (!pageResultsA || !pageResultsB) return 0;
+			return pageResultsB.length - pageResultsA.length;
+		});
+
+		for (const url of sortedUrls) {
+			const pageResults = productGroup.get(url);
+			if (!pageResults || pageResults.length === 0) continue;
+			const firstResult = pageResults[0];
+
+			// Skip if no results (shouldn't happen but TypeScript safety)
+			if (!firstResult) continue;
+
+			// Page header with link and hit count
+			const pageTitle = firstResult.heading1 || firstResult.source_page_title;
+			const hitCount = pageResults.length > 1 ? ` (${pageResults.length} results)` : '';
+			lines.push(`### Results from [${escapeMarkdown(pageTitle)}](${url})${hitCount}`);
+			lines.push('');
+
+			// Add each excerpt from this page
+			for (const result of pageResults) {
+				lines.push(formatExcerpt(result));
+			}
+		}
+	}
+
+	// Add suggestion to use doc fetch tool
+	lines.push('---');
+	lines.push(`Use the "${DOC_FETCH_CONFIG.name}" tool to download a specific document.`);
+
+	return lines.join('\n');
+}

package/src/index.ts

@@ -13,6 +13,8 @@ export * from './space-info.js';
 export * from './space-files.js';
 export * from './user-summary.js';
 export * from './paper-summary.js';
+export * from './docs-search/docs-semantic-search.js';
+export * from './docs-search/doc-fetch.js';
 
 // Export tool IDs for external use - these are the canonical tool identifiers
 export * from './tool-ids.js';

package/src/tool-ids.ts

@@ -15,6 +15,8 @@ import {
 	SPACE_FILES_TOOL_CONFIG,
 	USER_SUMMARY_PROMPT_CONFIG,
 	PAPER_SUMMARY_PROMPT_CONFIG,
+	DOCS_SEMANTIC_SEARCH_CONFIG,
+	DOC_FETCH_CONFIG,
 } from './index.js';
 
 // Extract tool IDs from their configs (single source of truth)
@@ -27,9 +29,12 @@ export const DATASET_DETAIL_TOOL_ID = DATASET_DETAIL_TOOL_CONFIG.name;
 export const DUPLICATE_SPACE_TOOL_ID = DUPLICATE_SPACE_TOOL_CONFIG.name;
 export const SPACE_INFO_TOOL_ID = SPACE_INFO_TOOL_CONFIG.name;
 export const SPACE_FILES_TOOL_ID = SPACE_FILES_TOOL_CONFIG.name;
+export const DOCS_SEMANTIC_SEARCH_TOOL_ID = DOCS_SEMANTIC_SEARCH_CONFIG.name;
+export const DOC_FETCH_TOOL_ID = DOC_FETCH_CONFIG.name;
 export const USER_SUMMARY_PROMPT_ID = USER_SUMMARY_PROMPT_CONFIG.name;
 export const PAPER_SUMMARY_PROMPT_ID = PAPER_SUMMARY_PROMPT_CONFIG.name;
 
+
 // Complete list of all built-in tool IDs
 export const ALL_BUILTIN_TOOL_IDS = [
 	SPACE_SEARCH_TOOL_ID,
@@ -41,12 +46,15 @@ export const ALL_BUILTIN_TOOL_IDS = [
 	DUPLICATE_SPACE_TOOL_ID,
 	SPACE_INFO_TOOL_ID,
 	SPACE_FILES_TOOL_ID,
+	DOCS_SEMANTIC_SEARCH_TOOL_ID,
+	DOC_FETCH_TOOL_ID,
 ] as const;
 // Grouped tool IDs for bouquet configurations
 export const TOOL_ID_GROUPS = {
-	search: [SPACE_SEARCH_TOOL_ID, MODEL_SEARCH_TOOL_ID, DATASET_SEARCH_TOOL_ID, PAPER_SEARCH_TOOL_ID] as const,
+	search: [SPACE_SEARCH_TOOL_ID, MODEL_SEARCH_TOOL_ID, DATASET_SEARCH_TOOL_ID, PAPER_SEARCH_TOOL_ID, DOCS_SEMANTIC_SEARCH_TOOL_ID] as const,
 	spaces: [SPACE_SEARCH_TOOL_ID, DUPLICATE_SPACE_TOOL_ID, SPACE_INFO_TOOL_ID, SPACE_FILES_TOOL_ID] as const,
 	detail: [MODEL_DETAIL_TOOL_ID, DATASET_DETAIL_TOOL_ID] as const,
+	docs: [DOCS_SEMANTIC_SEARCH_TOOL_ID, DOC_FETCH_TOOL_ID] as const,
 	hf_api: [
 		SPACE_SEARCH_TOOL_ID,
 		MODEL_SEARCH_TOOL_ID,
@@ -65,20 +73,3 @@ export type BuiltinToolId = (typeof ALL_BUILTIN_TOOL_IDS)[number];
 export function isValidBuiltinToolId(toolId: string): toolId is BuiltinToolId {
 	return (ALL_BUILTIN_TOOL_IDS as readonly string[]).includes(toolId);
 }
-
-// Helper to get tool description from configs
-export function getToolDescription(toolId: BuiltinToolId): string {
-	const configs = {
-		[SPACE_SEARCH_TOOL_ID]: SEMANTIC_SEARCH_TOOL_CONFIG,
-		[MODEL_SEARCH_TOOL_ID]: MODEL_SEARCH_TOOL_CONFIG,
-		[MODEL_DETAIL_TOOL_ID]: MODEL_DETAIL_TOOL_CONFIG,
-		[PAPER_SEARCH_TOOL_ID]: PAPER_SEARCH_TOOL_CONFIG,
-		[DATASET_SEARCH_TOOL_ID]: DATASET_SEARCH_TOOL_CONFIG,
-		[DATASET_DETAIL_TOOL_ID]: DATASET_DETAIL_TOOL_CONFIG,
-		[DUPLICATE_SPACE_TOOL_ID]: DUPLICATE_SPACE_TOOL_CONFIG,
-		[SPACE_INFO_TOOL_ID]: SPACE_INFO_TOOL_CONFIG,
-		[SPACE_FILES_TOOL_ID]: SPACE_FILES_TOOL_CONFIG,
-	} as const;
-
-	return configs[toolId]?.description || `Tool: ${toolId}`;
-}