agent-search-mcp 2.1.0

package/dist/tools/fetch-tools.js

@@ -0,0 +1,200 @@
+import { z } from 'zod';
+import { logger } from '../infrastructure/index.js';
+/**
+ * Extract GitHub README content from a repository URL.
+ */
+export async function fetchGithubReadme(url) {
+    try {
+        // Parse GitHub URL to extract owner/repo
+        const githubMatch = url.match(/github\.com\/([^\/]+)\/([^\/]+)/);
+        if (!githubMatch) {
+            throw new Error('Invalid GitHub URL');
+        }
+        const [, owner, repo] = githubMatch;
+        const cleanRepo = repo.replace(/\.git$/, '');
+        // Try common README filenames
+        const readmeFiles = ['README.md', 'readme.md', 'Readme.md', 'README.MD', 'README'];
+        for (const filename of readmeFiles) {
+            try {
+                const rawUrl = `https://raw.githubusercontent.com/${owner}/${cleanRepo}/main/${filename}`;
+                const response = await fetch(rawUrl, {
+                    signal: AbortSignal.timeout(10000),
+                });
+                if (response.ok) {
+                    const content = await response.text();
+                    return `# ${owner}/${cleanRepo}\n\n${content}`;
+                }
+            }
+            catch {
+                // Try master branch if main fails
+                try {
+                    const rawUrl = `https://raw.githubusercontent.com/${owner}/${cleanRepo}/master/${filename}`;
+                    const response = await fetch(rawUrl, {
+                        signal: AbortSignal.timeout(10000),
+                    });
+                    if (response.ok) {
+                        const content = await response.text();
+                        return `# ${owner}/${cleanRepo}\n\n${content}`;
+                    }
+                }
+                catch {
+                    // Continue to next filename
+                }
+            }
+        }
+        throw new Error('README not found');
+    }
+    catch (error) {
+        logger.error({ err: error instanceof Error ? error.message : String(error) }, 'Failed to fetch GitHub README');
+        throw error;
+    }
+}
+/**
+ * Extract CSDN article content.
+ */
+export async function fetchCsdnArticle(url) {
+    try {
+        const response = await fetch(url, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+            },
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        const html = await response.text();
+        // Simple extraction: find article content between common markers
+        const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i) ||
+            html.match(/<div[^>]*class="[^"]*article[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
+        if (articleMatch) {
+            // Basic HTML to text conversion
+            let content = articleMatch[1]
+                .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+                .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+                .replace(/<[^>]+>/g, '\n')
+                .replace(/\n{3,}/g, '\n\n')
+                .trim();
+            return content;
+        }
+        throw new Error('Article content not found');
+    }
+    catch (error) {
+        logger.error({ err: error instanceof Error ? error.message : String(error) }, 'Failed to fetch CSDN article');
+        throw error;
+    }
+}
+/**
+ * Extract Juejin article content.
+ */
+export async function fetchJuejinArticle(url) {
+    try {
+        // Extract article ID from URL
+        const idMatch = url.match(/post\/(\d+)/);
+        if (!idMatch) {
+            throw new Error('Invalid Juejin URL');
+        }
+        const articleId = idMatch[1];
+        const apiUrl = `https://api.juejin.cn/content_api/v1/article/detail?article_id=${articleId}`;
+        const response = await fetch(apiUrl, {
+            signal: AbortSignal.timeout(10000),
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        const data = await response.json();
+        if (data.err_no !== 0 || !data.data) {
+            throw new Error(data.err_msg || 'Failed to fetch article');
+        }
+        const article = data.data;
+        const content = article.article_info?.markdown_content || article.article_info?.content || '';
+        return `# ${article.article_info?.title || 'Juejin Article'}\n\n${content}`;
+    }
+    catch (error) {
+        logger.error({ err: error instanceof Error ? error.message : String(error) }, 'Failed to fetch Juejin article');
+        throw error;
+    }
+}
+export function setupFetchTools(server) {
+    // GitHub README tool
+    server.tool('fetch_github_readme', 'Fetch README content from a GitHub repository', {
+        url: z.string().url('Must be a valid URL').describe('GitHub repository URL (e.g., https://github.com/owner/repo)'),
+    }, async ({ url }) => {
+        try {
+            const content = await fetchGithubReadme(url);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: content,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `Failed to fetch GitHub README: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+    // CSDN article tool
+    server.tool('fetch_csdn_article', 'Fetch content from a CSDN blog article', {
+        url: z.string().url('Must be a valid URL').describe('CSDN article URL'),
+    }, async ({ url }) => {
+        try {
+            const content = await fetchCsdnArticle(url);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: content,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `Failed to fetch CSDN article: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+    // Juejin article tool
+    server.tool('fetch_juejin_article', 'Fetch content from a Juejin article', {
+        url: z.string().url('Must be a valid URL').describe('Juejin article URL'),
+    }, async ({ url }) => {
+        try {
+            const content = await fetchJuejinArticle(url);
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: content,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `Failed to fetch Juejin article: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/free-extract.js

@@ -0,0 +1,43 @@
+import { z } from 'zod';
+import { validateUrl } from '../infrastructure/url-validator.js';
+export function registerFreeExtract(server) {
+    server.tool('free_extract', `Extract full content from a URL. Returns clean markdown.
+
+Best for: Reading a specific page found in search results.
+Not recommended for: Bulk extraction — use search first.`, {
+        url: z.string().describe('URL to extract'),
+        max_length: z.number().optional().default(5000).describe('Max characters to return'),
+    }, async ({ url, max_length }) => {
+        // SSRF 防护
+        const validation = validateUrl(url);
+        if (!validation.valid) {
+            return {
+                content: [{ type: 'text', text: `Error: ${validation.error}` }],
+                isError: true,
+            };
+        }
+        try {
+            // 使用 Jina Reader
+            const res = await fetch(`https://r.jina.ai/${url}`, {
+                headers: { 'Accept': 'text/markdown' },
+                signal: AbortSignal.timeout(10000),
+            });
+            if (!res.ok) {
+                return {
+                    content: [{ type: 'text', text: `Error: HTTP ${res.status}` }],
+                    isError: true,
+                };
+            }
+            const content = await res.text();
+            return {
+                content: [{ type: 'text', text: content.slice(0, max_length) }],
+            };
+        }
+        catch (error) {
+            return {
+                content: [{ type: 'text', text: `Error: ${error instanceof Error ? error.message : String(error)}` }],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/free-search-advanced.js

@@ -0,0 +1,40 @@
+import { z } from 'zod';
+import { searchWithFallback } from './free-search.js';
+export function registerFreeSearchAdvanced(server) {
+    server.tool('free_search_advanced', `Advanced search with filters and quality control.
+
+Best for: Date ranges, domain filtering, high-confidence only, Chinese content.
+Not recommended for: Simple queries — use free_search instead.`, {
+        query: z.string().describe('Search query'),
+        count: z.number().optional().default(5).describe('Number of results (1-20)'),
+        min_confidence: z.number().min(1).max(3).optional().default(1)
+            .describe('Only return results verified by N+ sources'),
+        time_range: z.enum(['day', 'week', 'month', 'year']).optional()
+            .describe('Filter by recency'),
+        language: z.enum(['auto', 'en', 'zh']).optional().default('auto')
+            .describe('Language preference'),
+        include_domains: z.array(z.string()).optional()
+            .describe('Only search these domains'),
+        exclude_domains: z.array(z.string()).optional()
+            .describe('Exclude these domains'),
+    }, async (input) => {
+        try {
+            const results = await searchWithFallback({
+                query: input.query,
+                count: input.count,
+                engines: ['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily'],
+                minConfidence: input.min_confidence,
+                language: input.language,
+                includeDomains: input.include_domains,
+                excludeDomains: input.exclude_domains,
+            });
+            return { content: [{ type: 'text', text: JSON.stringify(results, null, 2) }] };
+        }
+        catch (error) {
+            return {
+                content: [{ type: 'text', text: `Search failed: ${error instanceof Error ? error.message : 'Unknown error'}` }],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/free-search.js

@@ -0,0 +1,380 @@
+import { z } from 'zod';
+import { searchDuckDuckGo } from '../engines/duckduckgo.js';
+import { searchSogou } from '../engines/sogou.js';
+import { searchBing } from '../engines/bing.js';
+import { searchBaidu } from '../engines/baidu.js';
+import { BraveProvider } from '../engines/brave.js';
+import { TavilyProvider } from '../engines/tavily.js';
+import { searchExa } from '../engines/exa.js';
+import { dedupByUrl, dedupByTitle, filterLowQuality, scoreAndRank, formatResults } from '../aggregation/index.js';
+import { SearchCache, logger, HealthTracker, RateLimiter } from '../infrastructure/index.js';
+const SUPPORTED_ENGINES = ['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily', 'exa'];
+const FREE_ENGINES = ['duckduckgo', 'sogou', 'bing', 'baidu'];
+const PAID_ENGINES = ['brave', 'tavily', 'exa'];
+// Engine weights (higher = more trusted)
+const ENGINE_WEIGHTS = {
+    duckduckgo: 0.85,
+    sogou: 0.8,
+    bing: 0.9,
+    baidu: 0.75,
+    brave: 0.95,
+    tavily: 0.9,
+    exa: 0.92,
+};
+// Infrastructure singletons
+const cache = new SearchCache();
+const healthTracker = new HealthTracker();
+const rateLimiter = new RateLimiter();
+// ─── Engine provider mapping (from ddgs pattern) ──────────────────────────
+// DDG uses Bing as backend, so we track providers to avoid duplicate queries
+const PROVIDER_MAP = {
+    duckduckgo: 'bing',
+    sogou: 'sogou',
+    bing: 'bing',
+    baidu: 'baidu',
+    brave: 'brave',
+    tavily: 'tavily',
+    exa: 'exa',
+};
+/**
+ * Get unique providers from engine list.
+ * From ddgs: same provider only searches once.
+ */
+function getUniqueProviders(engines) {
+    const seenProviders = new Set();
+    const unique = [];
+    for (const engine of engines) {
+        const provider = PROVIDER_MAP[engine] || engine;
+        if (!seenProviders.has(provider)) {
+            seenProviders.add(provider);
+            unique.push(engine);
+        }
+    }
+    return unique;
+}
+/**
+ * Search a single engine with health check, rate limiting, and retry logic.
+ */
+async function searchEngine(engine, query, limit, maxRetries = 2) {
+    // Skip unhealthy providers
+    if (!healthTracker.isHealthy(engine)) {
+        logger.warn({ engine }, 'Skipping unhealthy provider');
+        return [];
+    }
+    // Rate limit before making the request
+    await rateLimiter.waitForSlot(engine);
+    let lastError = null;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        const startTime = Date.now();
+        try {
+            let results;
+            switch (engine) {
+                case 'duckduckgo':
+                    results = await searchDuckDuckGo(query, limit);
+                    break;
+                case 'sogou':
+                    results = await searchSogou(query, limit);
+                    break;
+                case 'bing':
+                    results = await searchBing(query, limit);
+                    break;
+                case 'baidu':
+                    results = await searchBaidu(query, limit);
+                    break;
+                case 'brave':
+                    results = await new BraveProvider().search(query, limit);
+                    break;
+                case 'tavily':
+                    results = await new TavilyProvider().search(query, limit);
+                    break;
+                case 'exa':
+                    results = await searchExa({ query, count: limit, apiKey: process.env.EXA_API_KEY });
+                    break;
+                default:
+                    return [];
+            }
+            const latency = Date.now() - startTime;
+            healthTracker.recordSuccess(engine, latency);
+            logger.info({ engine, latency, count: results.length, attempt }, 'Search completed');
+            return results;
+        }
+        catch (err) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+            const latency = Date.now() - startTime;
+            // Check if this is a retryable error (network, timeout, 5xx)
+            const isRetryable = isRetryableError(lastError);
+            if (attempt < maxRetries && isRetryable) {
+                // Exponential backoff: 500ms, 1000ms, 2000ms...
+                const delay = Math.min(500 * Math.pow(2, attempt), 5000);
+                logger.warn({ engine, attempt, delay, err: lastError.message }, 'Retryable error, retrying...');
+                await new Promise(resolve => setTimeout(resolve, delay));
+                continue;
+            }
+            // Non-retryable or max retries exceeded
+            healthTracker.recordFailure(engine);
+            logger.error({ engine, latency, attempt, err: lastError.message }, 'Search failed');
+            return [];
+        }
+    }
+    // Should not reach here, but just in case
+    return [];
+}
+/**
+ * Check if an error is retryable (network, timeout, 5xx).
+ */
+function isRetryableError(err) {
+    const msg = err.message.toLowerCase();
+    // Network errors
+    if (msg.includes('econnreset') || msg.includes('econnrefused') ||
+        msg.includes('etimedout') || msg.includes('network')) {
+        return true;
+    }
+    // Timeout
+    if (msg.includes('timeout') || msg.includes('abort')) {
+        return true;
+    }
+    // HTTP 5xx errors (but not 501 Not Implemented)
+    if (msg.includes('http 5') && !msg.includes('http 501')) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Check if a paid engine has its API key configured.
+ */
+function hasApiKey(engine) {
+    switch (engine) {
+        case 'brave':
+            return !!process.env.BRAVE_API_KEY;
+        case 'tavily':
+            return !!process.env.TAVILY_API_KEY;
+        case 'exa':
+            return !!process.env.EXA_API_KEY;
+        default:
+            return true; // free engines always available
+    }
+}
+// ─── Request collapsing ───────────────────────────────────────────────
+// Track in-flight requests to avoid duplicate concurrent calls
+const pendingRequests = new Map();
+/**
+ * Generate cache key for request collapsing.
+ */
+function makeCollapseKey(options) {
+    const { query, count = 10, engines = [] } = options;
+    const sortedEngines = [...engines].sort().join(',');
+    return `${query}:${count}:${sortedEngines}`;
+}
+// ─── Core search logic (fused patterns from ddgs) ──────────────────────
+/**
+ * Search with provider dedup, batch concurrency, and early exit.
+ *
+ * Patterns from ddgs:
+ * 1. Provider dedup: same provider only searches once
+ * 2. Batch concurrency: search in batches to avoid rate limits
+ * 3. Early exit: stop when enough results collected
+ * 4. Frequency scoring: count how many engines returned each result
+ */
+export async function searchWithFallback(options) {
+    const collapseKey = makeCollapseKey(options);
+    // Check if same request is already in-flight
+    const pending = pendingRequests.get(collapseKey);
+    if (pending) {
+        logger.info({ query: options.query }, 'Request collapsing: reusing pending request');
+        return pending;
+    }
+    // Start new request and track it
+    const searchPromise = executeSearch(options);
+    pendingRequests.set(collapseKey, searchPromise);
+    // Clean up when done
+    searchPromise.finally(() => {
+        pendingRequests.delete(collapseKey);
+    });
+    return searchPromise;
+}
+/**
+ * Execute the actual search logic (internal).
+ */
+async function executeSearch(options) {
+    const { query, count = 10, engines: userEngines = ['duckduckgo', 'sogou'], minConfidence = 1, language, includeDomains, excludeDomains, } = options;
+    // Check cache first
+    const cacheKey = cache.makeKey(query, count, userEngines);
+    const cached = cache.get(cacheKey);
+    if (cached) {
+        logger.info({ query, count, engines: userEngines }, 'Cache hit');
+        return cached;
+    }
+    logger.info({ query, count, engines: userEngines }, 'Starting search');
+    // ── Step 1: Provider dedup (from ddgs) ──────────────────────────────
+    // Only search each provider once (e.g., DDG and Bing both use Bing backend)
+    const uniqueEngines = getUniqueProviders(userEngines);
+    logger.info({ engines: uniqueEngines }, 'After provider dedup');
+    // ── Step 2: Determine which engines to search ───────────────────────
+    // Phase 1: Free engines
+    const freeToSearch = uniqueEngines.filter(e => FREE_ENGINES.includes(e));
+    const allFree = FREE_ENGINES.filter(e => !uniqueEngines.includes(e));
+    const phase1Engines = [...freeToSearch, ...allFree];
+    // ── Step 3: Batch concurrency + early exit (from ddgs) ──────────────
+    // Adaptive batch size based on count and engine count
+    const BATCH_SIZE = Math.max(2, Math.min(phase1Engines.length, Math.ceil(count / 10) + 1));
+    const allResults = [];
+    const failures = [];
+    const searchedEngines = [];
+    // Batch 1: Free engines
+    logger.info({ engines: phase1Engines }, 'Phase 1: free engines (batch)');
+    for (let i = 0; i < phase1Engines.length; i += BATCH_SIZE) {
+        const batch = phase1Engines.slice(i, i + BATCH_SIZE);
+        const batchResults = await Promise.allSettled(batch.map(async (engine) => {
+            const results = await searchEngine(engine, query, count);
+            searchedEngines.push(engine);
+            return { engine, results };
+        }));
+        for (const result of batchResults) {
+            if (result.status === 'fulfilled') {
+                allResults.push(...result.value.results);
+            }
+            else {
+                failures.push({
+                    engine: 'unknown',
+                    message: result.reason?.message || 'Unknown error',
+                });
+            }
+        }
+        // Early exit: stop if we have enough results
+        if (allResults.length >= count * 1.5) {
+            logger.info({ count: allResults.length }, 'Early exit: enough results');
+            break;
+        }
+    }
+    logger.info({ count: allResults.length }, 'Phase 1 results');
+    // ── Step 4: Fallback to paid engines if not enough ───────────────────
+    if (allResults.length < count) {
+        const paidToSearch = uniqueEngines.filter(e => PAID_ENGINES.includes(e) && hasApiKey(e));
+        if (paidToSearch.length > 0) {
+            const remaining = Math.max(count - allResults.length, 1);
+            logger.info({ engines: paidToSearch, remaining }, 'Phase 2: paid engines');
+            const phase2Results = await Promise.allSettled(paidToSearch.map(async (engine) => {
+                const results = await searchEngine(engine, query, remaining);
+                searchedEngines.push(engine);
+                return { engine, results };
+            }));
+            for (const result of phase2Results) {
+                if (result.status === 'fulfilled') {
+                    allResults.push(...result.value.results);
+                }
+                else {
+                    failures.push({
+                        engine: 'unknown',
+                        message: result.reason?.message || 'Unknown error',
+                    });
+                }
+            }
+            logger.info({ got: allResults.length }, 'Phase 2 results');
+        }
+        else {
+            logger.info('Phase 2: no paid engines available');
+        }
+    }
+    // ── Step 5: Aggregation layer (fused from ddgs + our patterns) ──────
+    // 5a. Filter low-quality results (from ddgs)
+    const filtered = filterLowQuality(allResults);
+    // 5b. URL dedup with frequency counting
+    const { results: urlDeduped, frequencies } = dedupByUrl(filtered);
+    // 5c. Title dedup
+    const titleDeduped = dedupByTitle(urlDeduped);
+    // 5d. Score and rank with frequency bonus
+    let scored = scoreAndRank(titleDeduped, query, ENGINE_WEIGHTS, frequencies);
+    // ── Step 6: Post-search filters ─────────────────────────────────────
+    if (minConfidence > 1) {
+        scored = scored.filter(r => r.confidence >= minConfidence);
+    }
+    if (includeDomains && includeDomains.length > 0) {
+        scored = scored.filter(r => {
+            try {
+                const hostname = new URL(r.url).hostname;
+                return includeDomains.some(d => hostname.includes(d) || hostname.endsWith(d));
+            }
+            catch {
+                return false;
+            }
+        });
+    }
+    if (excludeDomains && excludeDomains.length > 0) {
+        scored = scored.filter(r => {
+            try {
+                const hostname = new URL(r.url).hostname;
+                return !excludeDomains.some(d => hostname.includes(d) || hostname.endsWith(d));
+            }
+            catch {
+                return true;
+            }
+        });
+    }
+    // ── Step 7: Format output with security processing ──────────────────
+    const formatted = formatResults(scored);
+    const response = {
+        query,
+        engines: userEngines,
+        ...formatted,
+        ...(failures.length > 0
+            ? { partialFailures: failures }
+            : {}),
+    };
+    // ── Step 8: Async cache write (from ddgs) ───────────────────────────
+    // Don't block the response - write cache in background
+    setImmediate(() => {
+        try {
+            cache.set(cacheKey, response);
+            logger.info({ total: response.meta.total }, 'Search complete');
+        }
+        catch (err) {
+            logger.error({ err }, 'Cache write failed');
+        }
+    });
+    return response;
+}
+// ─── Tool registration ──────────────────────────────────────────────────
+// Export the health tracker instance so index.ts can use the same singleton
+export { healthTracker };
+export function setupFreeSearchTool(server) {
+    server.tool('free_search', 'Search the web with automatic fallback between free and paid engines. ' +
+        'Phase 1: DuckDuckGo + Sogou + Bing + Baidu (free, no key required). ' +
+        'Phase 2: Brave + Tavily + Exa (paid, requires BRAVE_API_KEY / TAVILY_API_KEY / EXA_API_KEY env vars). ' +
+        'All results are deduplicated, scored, and ranked. ' +
+        'Results include security metadata to protect against prompt injection.', {
+        query: z.string().min(1, 'Search query must not be empty'),
+        limit: z.number().int().min(1).max(50).default(10).describe('Number of results to return (1-50)'),
+        engines: z.array(z.enum(['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily', 'exa']))
+            .min(1)
+            .default(['duckduckgo', 'sogou'])
+            .describe('Search engines to use (default: all free engines)'),
+    }, async ({ query, limit = 10, engines: userEngines }) => {
+        try {
+            const results = await searchWithFallback({
+                query,
+                count: limit,
+                engines: userEngines,
+            });
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify(results, null, 2),
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            logger.error({ err: error instanceof Error ? error.message : String(error) }, 'Search tool execution failed');
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `Search failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/dist/tools/health.js

@@ -0,0 +1,9 @@
+export function registerHealth(server, health) {
+    server.resource('health', 'search://health', async () => ({
+        contents: [{
+                uri: 'search://health',
+                mimeType: 'application/json',
+                text: JSON.stringify(health.getHealth(), null, 2),
+            }]
+    }));
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};