mcp-searxng 1.7.0 → 1.7.1

package/README.md

@@ -12,9 +12,12 @@
 [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/ihor-sokoliuk/mcp-searxng/badge)](https://scorecard.dev/viewer/?uri=github.com/ihor-sokoliuk/mcp-searxng)
 [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/13143/badge)](https://www.bestpractices.dev/projects/13143)
 [![mcp-searxng MCP server](https://glama.ai/mcp/servers/ihor-sokoliuk/mcp-searxng/badges/score.svg)](https://glama.ai/mcp/servers/ihor-sokoliuk/mcp-searxng)
+[![GitHub MCP Registry](https://img.shields.io/badge/GitHub_MCP_Registry-listed-2da44e?style=flat-square&logo=github&logoColor=white)](https://github.com/mcp/ihor-sokoliuk/mcp-searxng)
 
 An [MCP server](https://modelcontextprotocol.io/introduction) that integrates the [SearXNG](https://docs.searxng.org) API, giving AI assistants web search capabilities.
 
+✨ Featured in the [GitHub MCP Registry](https://github.com/mcp/ihor-sokoliuk/mcp-searxng).
+
 </div>
 
 ## Quick Start

package/dist/proxy.d.ts

@@ -1,4 +1,7 @@
+import * as dns from "node:dns";
 import { Agent, ProxyAgent } from "undici";
+type LookupCallback = (err: NodeJS.ErrnoException | null, address: string | dns.LookupAddress[], family?: number) => void;
+export declare function createUrlReaderLookup(): (hostname: string, options: dns.LookupOptions, callback: LookupCallback) => void;
 /**
  * Proxy configuration type for separating search and URL reader proxies.
  */
@@ -38,3 +41,11 @@ export type ProxyType = typeof ProxyType[keyof typeof ProxyType];
  */
 export declare function createProxyAgent(targetUrl?: string, type?: ProxyType): ProxyAgent | undefined;
 export declare function createDefaultAgent(): Agent | undefined;
+/**
+ * Returns a singleton undici Agent for direct `web_url_read` requests.
+ *
+ * Unlike the shared default agent, this is always created so the URL reader's
+ * DNS validation hook is present even when no system CA bundle is detected.
+ */
+export declare function createUrlReaderAgent(): Agent;
+export {};

package/dist/proxy.js

@@ -1,5 +1,38 @@
+import * as dns from "node:dns";
 import { Agent, ProxyAgent } from "undici";
+import { getHttpSecurityConfig } from "./http-security.js";
 import { getConnectOptions } from "./tls-config.js";
+import { createUrlSecurityPolicyDnsError, isPrivateAddress } from "./url-security.js";
+export function createUrlReaderLookup() {
+    return (hostname, options, callback) => {
+        if (getHttpSecurityConfig().allowPrivateUrls) {
+            dns.lookup(hostname, options, callback);
+            return;
+        }
+        dns.lookup(hostname, { ...options, all: true }, (error, addresses) => {
+            if (error) {
+                callback(error, options.all ? [] : "");
+                return;
+            }
+            if (addresses.length === 0) {
+                const notFound = new Error(`No DNS records found for ${hostname}`);
+                notFound.code = "ENOTFOUND";
+                callback(notFound, options.all ? [] : "");
+                return;
+            }
+            if (addresses.some(({ address }) => isPrivateAddress(address))) {
+                callback(createUrlSecurityPolicyDnsError(hostname), options.all ? [] : "");
+                return;
+            }
+            const selected = addresses[0];
+            if (options.all) {
+                callback(null, [selected]);
+                return;
+            }
+            callback(null, selected.address, selected.family);
+        });
+    };
+}
 /**
  * Checks if a target URL should bypass the proxy based on NO_PROXY environment variable.
  *
@@ -203,6 +236,7 @@ export function createProxyAgent(targetUrl, type) {
  */
 let _defaultAgentInitialized = false;
 let _defaultAgent;
+let _urlReaderAgent;
 export function createDefaultAgent() {
     if (!_defaultAgentInitialized) {
         _defaultAgentInitialized = true;
@@ -213,3 +247,20 @@ export function createDefaultAgent() {
     }
     return _defaultAgent;
 }
+/**
+ * Returns a singleton undici Agent for direct `web_url_read` requests.
+ *
+ * Unlike the shared default agent, this is always created so the URL reader's
+ * DNS validation hook is present even when no system CA bundle is detected.
+ */
+export function createUrlReaderAgent() {
+    if (!_urlReaderAgent) {
+        _urlReaderAgent = new Agent({
+            connect: {
+                ...getConnectOptions(),
+                lookup: createUrlReaderLookup(),
+            },
+        });
+    }
+    return _urlReaderAgent;
+}

package/dist/url-reader.js

@@ -1,68 +1,14 @@
-import { isIP } from "node:net";
 import { NodeHtmlMarkdown } from "node-html-markdown";
 import { fetch as undiciFetch } from "undici";
-import { createProxyAgent, createDefaultAgent, ProxyType } from "./proxy.js";
+import { createProxyAgent, createUrlReaderAgent, ProxyType } from "./proxy.js";
 import { logMessage } from "./logging.js";
 import { urlCache } from "./cache.js";
-import { getHttpSecurityConfig } from "./http-security.js";
+import { assertUrlAllowed, isUrlSecurityPolicyDnsError } from "./url-security.js";
 import { createURLFormatError, createURLSecurityPolicyError, createNetworkError, createServerError, createContentError, createConversionError, createTimeoutError, createEmptyContentWarning, createUnexpectedError } from "./error-handler.js";
 const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
 const MAX_REDIRECTS = 5;
 export const DEFAULT_MAX_CONTENT_LENGTH_BYTES = 5 * 1024 * 1024;
 const HEAD_TIMEOUT_CAP_MS = 3000;
-function isPrivateHostname(hostname) {
-    const lower = hostname.toLowerCase().replace(/\.+$/, "");
-    return lower === "localhost" || lower.endsWith(".localhost");
-}
-function isPrivateIpv4(hostname) {
-    if (isIP(hostname) !== 4) {
-        return false;
-    }
-    return (hostname.startsWith("0.") ||
-        hostname.startsWith("10.") ||
-        hostname.startsWith("127.") ||
-        hostname.startsWith("192.168.") ||
-        /^172\.(1[6-9]|2\d|3[0-1])\./.test(hostname) ||
-        hostname.startsWith("169.254."));
-}
-function isPrivateIPv6(hostname) {
-    // url.hostname wraps IPv6 in brackets (e.g. "[::1]") — strip them first
-    const addr = (hostname.startsWith("[") && hostname.endsWith("]")
-        ? hostname.slice(1, -1)
-        : hostname).toLowerCase();
-    if (isIP(addr) !== 6)
-        return false;
-    if (addr === "::1")
-        return true; // loopback
-    if (addr === "::")
-        return true; // unspecified
-    if (/^f[cd]/i.test(addr))
-        return true; // ULA fc00::/7
-    if (/^fe[89ab][0-9a-f]:/i.test(addr))
-        return true; // link-local fe80::/10
-    // IPv4-mapped ::ffff:<ipv4> — delegate to the IPv4 check
-    const mapped = addr.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
-    if (mapped)
-        return isPrivateIpv4(mapped[1]);
-    // IPv4-mapped ::ffff:<hhhh>:<hhhh> — convert the hex segments to dotted decimal
-    const hexMapped = addr.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
-    if (hexMapped) {
-        const high = parseInt(hexMapped[1], 16);
-        const low = parseInt(hexMapped[2], 16);
-        const ipv4 = `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`;
-        return isPrivateIpv4(ipv4);
-    }
-    return false;
-}
-function assertUrlAllowed(url) {
-    const security = getHttpSecurityConfig();
-    if (security.allowPrivateUrls) {
-        return;
-    }
-    if (isPrivateHostname(url.hostname) || isPrivateIpv4(url.hostname) || isPrivateIPv6(url.hostname)) {
-        throw createURLSecurityPolicyError(url.toString());
-    }
-}
 function isRedirectResponse(response) {
     return REDIRECT_STATUS_CODES.has(response.status);
 }
@@ -186,6 +132,9 @@ export async function checkContentLength(mcpServer, url, timeoutMs, dispatcher,
         return Number.isNaN(parsed) || parsed < 0 ? null : parsed;
     }
     catch (error) {
+        if (isUrlSecurityPolicyDnsError(error)) {
+            throw createURLSecurityPolicyError(url);
+        }
         logMessage(mcpServer, "warning", `HEAD check failed (proceeding with GET): ${error.message}`);
         return null;
     }
@@ -211,6 +160,45 @@ function createContentTooLargeMessage(contentLength, maxBytes) {
     return (`Content too large: server reports Content-Length of ${sizeMB} MB (limit: ${limitMB} MB). ` +
         `Try using readHeadings or section to fetch only the relevant parts.`);
 }
+function concatenateChunks(chunks, totalBytes) {
+    const result = new Uint8Array(totalBytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return result;
+}
+async function readResponseBodyWithLimit(response, maxBytes) {
+    if (response.body === null) {
+        return { exceeded: false, text: "", bytesRead: 0 };
+    }
+    const reader = response.body.getReader();
+    const chunks = [];
+    let bytesRead = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            if (!value) {
+                continue;
+            }
+            bytesRead += value.byteLength;
+            if (bytesRead > maxBytes) {
+                await reader.cancel();
+                return { exceeded: true, bytesRead };
+            }
+            chunks.push(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const bodyBytes = concatenateChunks(chunks, bytesRead);
+    return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead };
+}
 export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 10000, paginationOptions = {}) {
     const startTime = Date.now();
     logMessage(mcpServer, "info", `Fetching URL: ${url}`);
@@ -258,7 +246,7 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
             for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
                 // Add proxy or default dispatcher (includes system CA certs for TLS)
                 const proxyAgent = createProxyAgent(currentUrl.toString(), ProxyType.URL_READER);
-                const dispatcher = proxyAgent ?? createDefaultAgent();
+                const dispatcher = proxyAgent ?? createUrlReaderAgent();
                 usedDispatcher = !!dispatcher;
                 const currentRequestOptions = {
                     ...requestOptions,
@@ -294,6 +282,9 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
             if (error.name === 'MCPSearXNGError') {
                 throw error;
             }
+            if (isUrlSecurityPolicyDnsError(error)) {
+                throw createURLSecurityPolicyError(currentUrl.toString());
+            }
             const context = {
                 url: currentUrl.toString(),
                 proxyAgent: usedDispatcher,
@@ -304,7 +295,10 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
         if (!response.ok) {
             let responseBody;
             try {
-                responseBody = await response.text();
+                const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+                responseBody = bodyRead.exceeded
+                    ? createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes)
+                    : bodyRead.text;
             }
             catch {
                 responseBody = '[Could not read response body]';
@@ -315,7 +309,11 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
         // Retrieve HTML content
         let htmlContent;
         try {
-            htmlContent = await response.text();
+            const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+            if (bodyRead.exceeded) {
+                return createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes);
+            }
+            htmlContent = bodyRead.text;
         }
         catch (error) {
             throw createContentError(`Failed to read website content: ${error.message || 'Unknown error reading content'}`, url);

package/dist/url-security.d.ts

@@ -0,0 +1,8 @@
+export declare const URL_SECURITY_POLICY_DNS_ERROR = "URLSecurityPolicyDnsError";
+export declare function isPrivateHostname(hostname: string): boolean;
+export declare function isPrivateIpv4(hostname: string): boolean;
+export declare function isPrivateIPv6(hostname: string): boolean;
+export declare function isPrivateAddress(address: string): boolean;
+export declare function assertUrlAllowed(url: URL): void;
+export declare function createUrlSecurityPolicyDnsError(hostname: string): NodeJS.ErrnoException;
+export declare function isUrlSecurityPolicyDnsError(error: unknown): boolean;

package/dist/url-security.js

@@ -0,0 +1,79 @@
+import { isIP } from "node:net";
+import { createURLSecurityPolicyError } from "./error-handler.js";
+import { getHttpSecurityConfig } from "./http-security.js";
+export const URL_SECURITY_POLICY_DNS_ERROR = "URLSecurityPolicyDnsError";
+export function isPrivateHostname(hostname) {
+    const lower = hostname.toLowerCase().replace(/\.+$/, "");
+    return lower === "localhost" || lower.endsWith(".localhost");
+}
+export function isPrivateIpv4(hostname) {
+    if (isIP(hostname) !== 4) {
+        return false;
+    }
+    return (hostname.startsWith("0.") ||
+        hostname.startsWith("10.") ||
+        hostname.startsWith("127.") ||
+        hostname.startsWith("192.168.") ||
+        /^172\.(1[6-9]|2\d|3[0-1])\./.test(hostname) ||
+        hostname.startsWith("169.254."));
+}
+export function isPrivateIPv6(hostname) {
+    // url.hostname wraps IPv6 in brackets (e.g. "[::1]") - strip them first
+    const addr = (hostname.startsWith("[") && hostname.endsWith("]")
+        ? hostname.slice(1, -1)
+        : hostname).toLowerCase();
+    if (isIP(addr) !== 6)
+        return false;
+    if (addr === "::1")
+        return true; // loopback
+    if (addr === "::")
+        return true; // unspecified
+    if (/^f[cd]/i.test(addr))
+        return true; // ULA fc00::/7
+    if (/^fe[89ab][0-9a-f]:/i.test(addr))
+        return true; // link-local fe80::/10
+    // IPv4-mapped ::ffff:<ipv4> - delegate to the IPv4 check
+    const mapped = addr.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
+    if (mapped)
+        return isPrivateIpv4(mapped[1]);
+    // IPv4-mapped ::ffff:<hhhh>:<hhhh> - convert the hex segments to dotted decimal
+    const hexMapped = addr.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
+    if (hexMapped) {
+        const high = parseInt(hexMapped[1], 16);
+        const low = parseInt(hexMapped[2], 16);
+        const ipv4 = `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`;
+        return isPrivateIpv4(ipv4);
+    }
+    return false;
+}
+export function isPrivateAddress(address) {
+    return isPrivateIpv4(address) || isPrivateIPv6(address);
+}
+export function assertUrlAllowed(url) {
+    const security = getHttpSecurityConfig();
+    if (security.allowPrivateUrls) {
+        return;
+    }
+    if (isPrivateHostname(url.hostname) || isPrivateIpv4(url.hostname) || isPrivateIPv6(url.hostname)) {
+        throw createURLSecurityPolicyError(url.toString());
+    }
+}
+export function createUrlSecurityPolicyDnsError(hostname) {
+    const error = new Error(`Resolved private address blocked by security policy for ${hostname}`);
+    error.name = URL_SECURITY_POLICY_DNS_ERROR;
+    error.code = URL_SECURITY_POLICY_DNS_ERROR;
+    return error;
+}
+export function isUrlSecurityPolicyDnsError(error) {
+    let current = error;
+    while (current) {
+        if (current.name === URL_SECURITY_POLICY_DNS_ERROR || current.code === URL_SECURITY_POLICY_DNS_ERROR) {
+            return true;
+        }
+        if (Array.isArray(current.errors) && current.errors.some(isUrlSecurityPolicyDnsError)) {
+            return true;
+        }
+        current = current.cause;
+    }
+    return false;
+}

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const packageVersion = "1.7.0";
+export declare const packageVersion = "1.7.1";

package/dist/version.js

@@ -1 +1 @@
-export const packageVersion = "1.7.0";
+export const packageVersion = "1.7.1";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcp-searxng",
-    "version": "1.7.0",
+    "version": "1.7.1",
     "mcpName": "io.github.ihor-sokoliuk/mcp-searxng",
     "description": "MCP server for SearXNG integration",
     "license": "MIT",