@kassol/mcp-searxng 1.0.3-custom.0

package/dist/url-reader.js

@@ -0,0 +1,276 @@
+import { isIP } from "node:net";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+import { createProxyAgent, createDefaultAgent, ProxyType } from "./proxy.js";
+import { logMessage } from "./logging.js";
+import { urlCache } from "./cache.js";
+import { getHttpSecurityConfig } from "./http-security.js";
+import { mergeHeaders, parseHeadersFromEnv } from "./headers.js";
+import { createURLFormatError, createURLSecurityPolicyError, createNetworkError, createServerError, createContentError, createConversionError, createTimeoutError, createEmptyContentWarning, createUnexpectedError } from "./error-handler.js";
+function isPrivateHostname(hostname) {
+    const lower = hostname.toLowerCase().replace(/\.+$/, "");
+    return lower === "localhost" || lower.endsWith(".localhost");
+}
+function isPrivateIpv4(hostname) {
+    if (isIP(hostname) !== 4) {
+        return false;
+    }
+    return (hostname.startsWith("10.") ||
+        hostname.startsWith("127.") ||
+        hostname.startsWith("192.168.") ||
+        /^172\.(1[6-9]|2\d|3[0-1])\./.test(hostname) ||
+        hostname.startsWith("169.254."));
+}
+function isPrivateIPv6(hostname) {
+    // url.hostname wraps IPv6 in brackets (e.g. "[::1]") — strip them first
+    const addr = (hostname.startsWith("[") && hostname.endsWith("]")
+        ? hostname.slice(1, -1)
+        : hostname).toLowerCase();
+    if (isIP(addr) !== 6)
+        return false;
+    if (addr === "::1")
+        return true; // loopback
+    if (addr === "::")
+        return true; // unspecified
+    if (/^f[cd]/i.test(addr))
+        return true; // ULA fc00::/7
+    if (/^fe[89ab][0-9a-f]:/i.test(addr))
+        return true; // link-local fe80::/10
+    // IPv4-mapped ::ffff:<ipv4> — delegate to the IPv4 check
+    const mapped = addr.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
+    if (mapped)
+        return isPrivateIpv4(mapped[1]);
+    return false;
+}
+function assertUrlAllowed(url) {
+    const security = getHttpSecurityConfig();
+    if (!security.harden || security.allowPrivateUrls) {
+        return;
+    }
+    if (isPrivateHostname(url.hostname) || isPrivateIpv4(url.hostname) || isPrivateIPv6(url.hostname)) {
+        throw createURLSecurityPolicyError(url.toString());
+    }
+}
+function applyCharacterPagination(content, startChar = 0, maxLength) {
+    if (startChar >= content.length) {
+        return "";
+    }
+    const start = Math.max(0, startChar);
+    const end = maxLength ? Math.min(content.length, start + maxLength) : content.length;
+    return content.slice(start, end);
+}
+function escapeRegExp(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function extractSection(markdownContent, sectionHeading) {
+    const lines = markdownContent.split('\n');
+    const sectionRegex = new RegExp(`^#{1,6}\\s*.*${escapeRegExp(sectionHeading)}.*$`, 'i');
+    let startIndex = -1;
+    let currentLevel = 0;
+    // Find the section start
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (sectionRegex.test(line)) {
+            startIndex = i;
+            currentLevel = (line.match(/^#+/) || [''])[0].length;
+            break;
+        }
+    }
+    if (startIndex === -1) {
+        return "";
+    }
+    // Find the section end (next heading of same or higher level)
+    let endIndex = lines.length;
+    for (let i = startIndex + 1; i < lines.length; i++) {
+        const line = lines[i];
+        const match = line.match(/^#+/);
+        if (match && match[0].length <= currentLevel) {
+            endIndex = i;
+            break;
+        }
+    }
+    return lines.slice(startIndex, endIndex).join('\n');
+}
+function extractParagraphRange(markdownContent, range) {
+    const paragraphs = markdownContent.split('\n\n').filter(p => p.trim().length > 0);
+    // Parse range (e.g., "1-5", "3", "10-")
+    const rangeMatch = range.match(/^(\d+)(?:-(\d*))?$/);
+    if (!rangeMatch) {
+        return "";
+    }
+    const start = parseInt(rangeMatch[1]) - 1; // Convert to 0-based index
+    const endStr = rangeMatch[2];
+    if (start < 0 || start >= paragraphs.length) {
+        return "";
+    }
+    if (endStr === undefined) {
+        // Single paragraph (e.g., "3")
+        return paragraphs[start] || "";
+    }
+    else if (endStr === "") {
+        // Range to end (e.g., "10-")
+        return paragraphs.slice(start).join('\n\n');
+    }
+    else {
+        // Specific range (e.g., "1-5")
+        const end = parseInt(endStr);
+        return paragraphs.slice(start, end).join('\n\n');
+    }
+}
+function extractHeadings(markdownContent) {
+    const lines = markdownContent.split('\n');
+    const headings = lines.filter(line => /^#{1,6}\s/.test(line));
+    if (headings.length === 0) {
+        return "No headings found in the content.";
+    }
+    return headings.join('\n');
+}
+function applyPaginationOptions(markdownContent, options) {
+    let result = markdownContent;
+    // Apply heading extraction first if requested
+    if (options.readHeadings) {
+        return extractHeadings(result);
+    }
+    // Apply section extraction
+    if (options.section) {
+        result = extractSection(result, options.section);
+        if (result === "") {
+            return `Section "${options.section}" not found in the content.`;
+        }
+    }
+    // Apply paragraph range filtering
+    if (options.paragraphRange) {
+        result = extractParagraphRange(result, options.paragraphRange);
+        if (result === "") {
+            return `Paragraph range "${options.paragraphRange}" is invalid or out of bounds.`;
+        }
+    }
+    // Apply character-based pagination last
+    if (options.startChar !== undefined || options.maxLength !== undefined) {
+        result = applyCharacterPagination(result, options.startChar, options.maxLength);
+    }
+    return result;
+}
+export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 10000, paginationOptions = {}) {
+    const startTime = Date.now();
+    logMessage(mcpServer, "info", `Fetching URL: ${url}`);
+    // Check cache first
+    const cachedEntry = urlCache.get(url);
+    if (cachedEntry) {
+        logMessage(mcpServer, "info", `Using cached content for URL: ${url}`);
+        const result = applyPaginationOptions(cachedEntry.markdownContent, paginationOptions);
+        const duration = Date.now() - startTime;
+        logMessage(mcpServer, "info", `Processed cached URL: ${url} (${result.length} chars in ${duration}ms)`);
+        return result;
+    }
+    // Validate URL format
+    let parsedUrl;
+    try {
+        parsedUrl = new URL(url);
+    }
+    catch (error) {
+        logMessage(mcpServer, "error", `Invalid URL format: ${url}`);
+        throw createURLFormatError(url);
+    }
+    assertUrlAllowed(parsedUrl);
+    // Create an AbortController instance
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        // Prepare request options with proxy support
+        const requestOptions = {
+            signal: controller.signal,
+        };
+        // Add proxy or default dispatcher (includes system CA certs for TLS)
+        const proxyAgent = createProxyAgent(url, ProxyType.URL_READER);
+        const dispatcher = proxyAgent ?? createDefaultAgent();
+        if (dispatcher) {
+            requestOptions.dispatcher = dispatcher;
+        }
+        // Add User-Agent header if configured (URL_READER_USER_AGENT takes priority over USER_AGENT)
+        const userAgent = process.env.URL_READER_USER_AGENT || process.env.USER_AGENT;
+        if (userAgent) {
+            requestOptions.headers = {
+                ...requestOptions.headers,
+                'User-Agent': userAgent
+            };
+        }
+        const additionalHeaders = parseHeadersFromEnv("URL_READER_HEADERS");
+        if (Object.keys(additionalHeaders).length > 0) {
+            requestOptions.headers = mergeHeaders(requestOptions.headers, additionalHeaders);
+        }
+        let response;
+        try {
+            // Fetch the URL with the abort signal
+            response = await fetch(url, requestOptions);
+        }
+        catch (error) {
+            const context = {
+                url,
+                proxyAgent: !!dispatcher,
+                timeout: timeoutMs
+            };
+            throw createNetworkError(error, context);
+        }
+        if (!response.ok) {
+            let responseBody;
+            try {
+                responseBody = await response.text();
+            }
+            catch {
+                responseBody = '[Could not read response body]';
+            }
+            const context = { url };
+            throw createServerError(response.status, response.statusText, responseBody, context);
+        }
+        // Retrieve HTML content
+        let htmlContent;
+        try {
+            htmlContent = await response.text();
+        }
+        catch (error) {
+            throw createContentError(`Failed to read website content: ${error.message || 'Unknown error reading content'}`, url);
+        }
+        if (!htmlContent || htmlContent.trim().length === 0) {
+            throw createContentError("Website returned empty content.", url);
+        }
+        // Convert HTML to Markdown
+        let markdownContent;
+        try {
+            markdownContent = NodeHtmlMarkdown.translate(htmlContent);
+        }
+        catch (error) {
+            throw createConversionError(error, url, htmlContent);
+        }
+        if (!markdownContent || markdownContent.trim().length === 0) {
+            logMessage(mcpServer, "warning", `Empty content after conversion: ${url}`);
+            // DON'T cache empty/failed conversions - return warning directly
+            return createEmptyContentWarning(url, htmlContent.length, htmlContent);
+        }
+        // Only cache successful markdown conversion
+        urlCache.set(url, htmlContent, markdownContent);
+        // Apply pagination options
+        const result = applyPaginationOptions(markdownContent, paginationOptions);
+        const duration = Date.now() - startTime;
+        logMessage(mcpServer, "info", `Successfully fetched and converted URL: ${url} (${result.length} chars in ${duration}ms)`);
+        return result;
+    }
+    catch (error) {
+        if (error.name === "AbortError") {
+            logMessage(mcpServer, "error", `Timeout fetching URL: ${url} (${timeoutMs}ms)`);
+            throw createTimeoutError(timeoutMs, url);
+        }
+        // Re-throw our enhanced errors
+        if (error.name === 'MCPSearXNGError') {
+            logMessage(mcpServer, "error", `Error fetching URL: ${url} - ${error.message}`);
+            throw error;
+        }
+        // Catch any unexpected errors
+        logMessage(mcpServer, "error", `Unexpected error fetching URL: ${url}`, error);
+        const context = { url };
+        throw createUnexpectedError(error, context);
+    }
+    finally {
+        // Clean up the timeout to prevent memory leaks
+        clearTimeout(timeoutId);
+    }
+}

package/package.json

@@ -0,0 +1,72 @@
+{
+    "name": "@kassol/mcp-searxng",
+    "version": "1.0.3-custom.0",
+    "mcpName": "io.github.kassol/mcp-searxng",
+    "description": "MCP server for SearXNG integration",
+    "license": "MIT",
+    "author": "Ihor Sokoliuk (https://github.com/ihor-sokoliuk)",
+    "homepage": "https://github.com/kassol/mcp-searxng",
+    "bugs": "https://github.com/kassol/mcp-searxng/issues",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/kassol/mcp-searxng.git"
+    },
+    "keywords": [
+        "mcp",
+        "modelcontextprotocol",
+        "searxng",
+        "search",
+        "web-search",
+        "claude",
+        "ai",
+        "pagination",
+        "smithery",
+        "url-reader"
+    ],
+    "type": "module",
+    "bin": {
+        "mcp-searxng": "dist/index.js"
+    },
+    "main": "dist/index.js",
+    "files": [
+        "dist"
+    ],
+    "publishConfig": {
+        "access": "public"
+    },
+    "engines": {
+        "node": ">=20"
+    },
+    "scripts": {
+        "build": "tsc && shx chmod +x dist/*.js",
+        "watch": "tsc --watch",
+        "test": "cross-env SEARXNG_URL=https://test-searx.example.com tsx __tests__/run-all.ts",
+        "test:coverage": "cross-env SEARXNG_URL=https://test-searx.example.com c8 --reporter=text tsx __tests__/run-all.ts",
+        "bootstrap": "npm install && npm run build",
+        "inspector": "DANGEROUSLY_OMIT_AUTH=true npx @modelcontextprotocol/inspector node dist/index.js",
+        "lint": "eslint src __tests__",
+        "postversion": "TAG=$(node scripts/update-version.js | tail -1) && git add src/index.ts .mcp/server.json && git commit --amend --no-edit && git tag -f $TAG"
+    },
+    "dependencies": {
+        "@modelcontextprotocol/sdk": "1.29.0",
+        "@types/cors": "^2.8.19",
+        "@types/express": "^5.0.6",
+        "cors": "^2.8.6",
+        "express": "^5.2.1",
+        "node-html-markdown": "^2.0.0",
+        "undici": "^7.24.0"
+    },
+    "devDependencies": {
+        "@types/node": "^22.17.2",
+        "@types/supertest": "^7.2.0",
+        "@typescript-eslint/eslint-plugin": "^8.58.0",
+        "@typescript-eslint/parser": "^8.58.0",
+        "c8": "^11.0.0",
+        "cross-env": "^10.1.0",
+        "eslint": "^10.1.0",
+        "shx": "^0.4.0",
+        "supertest": "^7.2.2",
+        "tsx": "^4.21.0",
+        "typescript": "^5.8.3"
+    }
+}