mcp-searxng 0.9.2 → 0.10.0

package/README.md

@@ -49,12 +49,29 @@ An [MCP server](https://modelcontextprotocol.io/introduction) implementation tha
   - Example: `https://search.example.com`
 
 #### Optional
-- **`AUTH_USERNAME`** / **`AUTH_PASSWORD`**: HTTP Basic Auth credentials for password-protected instances
-- **`USER_AGENT`**: Custom User-Agent header (e.g., `MyBot/1.0`)
-- **`HTTP_PROXY`** / **`HTTPS_PROXY`**: Proxy URLs for routing traffic
+- **`AUTH_USERNAME`** / **`AUTH_PASSWORD`**: HTTP Basic Auth credentials for `searxng_web_search` (password-protected SearXNG instances)
+- **`USER_AGENT`**: Global default User-Agent header used by both `searxng_web_search` and `web_url_read` (e.g., `MyBot/1.0`)
+- **`URL_READER_USER_AGENT`**: Custom User-Agent specifically for the `web_url_read` tool (overrides `USER_AGENT` for URL reading requests)
+- **`HTTP_PROXY`** / **`HTTPS_PROXY`**: Global proxy URLs for routing traffic (fallback for both interfaces)
   - Format: `http://[username:password@]proxy.host:port`
 - **`NO_PROXY`**: Comma-separated bypass list (e.g., `localhost,.internal,example.com`)
 
+##### Interface-Specific Proxies (Optional)
+- **`SEARCH_HTTP_PROXY`** / **`SEARCH_HTTPS_PROXY`**: Proxy for `searxng_web_search` tool only
+- **`URL_READER_HTTP_PROXY`** / **`URL_READER_HTTPS_PROXY`**: Proxy for `web_url_read` tool only
+  - These take priority over `HTTP_PROXY`/`HTTPS_PROXY` for their respective interfaces
+
+#### Advanced Configuration
+
+```bash
+# Separate proxies for search and URL reading
+SEARCH_HTTP_PROXY=http://search-proxy:8080
+URL_READER_HTTP_PROXY=http://reader-proxy:8080
+
+# Custom user_agent for URL reader
+URL_READER_USER_AGENT="Mozilla/5.0 (compatible; Bot/1.0)"
+```
+
 ## Installation & Configuration
 
 ### [NPX](https://www.npmjs.com/package/mcp-searxng)
@@ -87,8 +104,11 @@ An [MCP server](https://modelcontextprotocol.io/introduction) implementation tha
         "AUTH_USERNAME": "your_username",
         "AUTH_PASSWORD": "your_password",
         "USER_AGENT": "MyBot/1.0",
-        "HTTP_PROXY": "http://proxy.company.com:8080",
-        "HTTPS_PROXY": "http://proxy.company.com:8080",
+        "URL_READER_USER_AGENT": "Mozilla/5.0 (compatible; MyBot/1.0)",
+        "SEARCH_HTTP_PROXY": "http://search-proxy.company.com:8080",
+        "URL_READER_HTTP_PROXY": "http://reader-proxy.company.com:8080",
+        "HTTP_PROXY": "http://global-proxy.company.com:8080",
+        "HTTPS_PROXY": "http://global-proxy.company.com:8080",
         "NO_PROXY": "localhost,127.0.0.1,.local,.internal"
       }
     }
@@ -132,15 +152,17 @@ npm install -g mcp-searxng
         "AUTH_USERNAME": "your_username",
         "AUTH_PASSWORD": "your_password",
         "USER_AGENT": "MyBot/1.0",
-        "HTTP_PROXY": "http://proxy.company.com:8080",
-        "HTTPS_PROXY": "http://proxy.company.com:8080",
+        "URL_READER_USER_AGENT": "Mozilla/5.0 (compatible; MyBot/1.0)",
+        "SEARCH_HTTP_PROXY": "http://search-proxy.company.com:8080",
+        "URL_READER_HTTP_PROXY": "http://reader-proxy.company.com:8080",
+        "HTTP_PROXY": "http://global-proxy.company.com:8080",
+        "HTTPS_PROXY": "http://global-proxy.company.com:8080",
         "NO_PROXY": "localhost,127.0.0.1,.local,.internal"
       }
     }
   }
 }
 ```
-
 </details>
 
 ### Docker
@@ -183,6 +205,11 @@ docker pull isokoliuk/mcp-searxng:latest
         "-e", "AUTH_USERNAME",
         "-e", "AUTH_PASSWORD",
         "-e", "USER_AGENT",
+        "-e", "URL_READER_USER_AGENT",
+        "-e", "SEARCH_HTTP_PROXY",
+        "-e", "SEARCH_HTTPS_PROXY",
+        "-e", "URL_READER_HTTP_PROXY",
+        "-e", "URL_READER_HTTPS_PROXY",
         "-e", "HTTP_PROXY",
         "-e", "HTTPS_PROXY",
         "-e", "NO_PROXY",
@@ -193,8 +220,11 @@ docker pull isokoliuk/mcp-searxng:latest
         "AUTH_USERNAME": "your_username",
         "AUTH_PASSWORD": "your_password",
         "USER_AGENT": "MyBot/1.0",
-        "HTTP_PROXY": "http://proxy.company.com:8080",
-        "HTTPS_PROXY": "http://proxy.company.com:8080",
+        "URL_READER_USER_AGENT": "Mozilla/5.0 (compatible; MyBot/1.0)",
+        "SEARCH_HTTP_PROXY": "http://search-proxy.company.com:8080",
+        "URL_READER_HTTP_PROXY": "http://reader-proxy.company.com:8080",
+        "HTTP_PROXY": "http://global-proxy.company.com:8080",
+        "HTTPS_PROXY": "http://global-proxy.company.com:8080",
         "NO_PROXY": "localhost,127.0.0.1,.local,.internal"
       }
     }
@@ -229,7 +259,10 @@ services:
       # - AUTH_USERNAME=your_username
       # - AUTH_PASSWORD=your_password
       # - USER_AGENT=MyBot/1.0
-      # - HTTP_PROXY=http://proxy.company.com:8080
+      # - URL_READER_USER_AGENT=Mozilla/5.0 (compatible; MyBot/1.0)
+      # - SEARCH_HTTP_PROXY=http://search-proxy.company.com:8080
+      # - URL_READER_HTTP_PROXY=http://reader-proxy.company.com:8080
+      # - HTTP_PROXY=http://global-proxy.company.com:8080
       # - HTTPS_PROXY=http://proxy.company.com:8080
       # - NO_PROXY=localhost,127.0.0.1,.local,.internal
 ```
@@ -275,6 +308,40 @@ MCP_HTTP_PORT=3000 SEARXNG_URL=http://localhost:8080 mcp-searxng
 curl http://localhost:3000/health
 ```
 
+## Troubleshooting
+
+### 403 Forbidden Error from SearXNG
+
+If you receive a `403 Forbidden` error when using `mcp-searxng`, it is likely because your SearXNG instance does not have JSON format enabled. This server requests results in JSON format (`format=json`), which must be explicitly allowed in SearXNG's configuration.
+
+**To fix this**, edit your SearXNG `settings.yml` (commonly located at `/etc/searxng/settings.yml`) and add `json` to the list of allowed formats:
+
+```yaml
+search:
+  formats:
+    - html
+    - json
+```
+
+After saving the file, restart your SearXNG instance. For example, if running with Docker:
+
+```bash
+docker restart searxng
+```
+
+You can verify JSON format is working by running:
+
+```bash
+curl 'http://localhost:8080/search?q=test&format=json'
+```
+
+You should receive a JSON response. If you still get a 403 error, double-check that:
+- The `settings.yml` file is correctly mounted into your Docker container
+- The YAML indentation is correct
+- The SearXNG instance was fully restarted after the configuration change
+
+For more details, see the [SearXNG settings documentation](https://docs.searxng.org/admin/settings/settings.html) and [this discussion](https://github.com/searxng/searxng/discussions/1789).
+
 ## Running evals
 
 ```bash

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-declare const packageVersion = "0.9.2";
+declare const packageVersion = "0.10.0";
 export { packageVersion };
 export declare function isWebUrlReadArgs(args: unknown): args is {
     url: string;

package/dist/index.js

@@ -11,7 +11,7 @@ import { createConfigResource, createHelpResource } from "./resources.js";
 import { createHttpServer } from "./http-server.js";
 import { validateEnvironment as validateEnv } from "./error-handler.js";
 // Use a static version string that will be updated by the version script
-const packageVersion = "0.9.2";
+const packageVersion = "0.10.0";
 // Export the version for use in other modules
 export { packageVersion };
 // Global state for logging level

package/dist/proxy.d.ts

@@ -1,16 +1,39 @@
 import { ProxyAgent } from "undici";
+/**
+ * Proxy configuration type for separating search and URL reader proxies.
+ */
+export declare const ProxyType: {
+    readonly SEARCH: "search";
+    readonly URL_READER: "url_reader";
+};
+export type ProxyType = typeof ProxyType[keyof typeof ProxyType];
 /**
  * Creates a proxy agent dispatcher for Node.js fetch API.
  *
  * Node.js fetch uses Undici under the hood, which requires a 'dispatcher' option
  * instead of 'agent'. This function creates a ProxyAgent compatible with fetch.
  *
- * Environment variables checked (in order):
- * - HTTP_PROXY / http_proxy: For HTTP requests
- * - HTTPS_PROXY / https_proxy: For HTTPS requests
+ * Environment variables checked (in order, depending on URL protocol):
+ * - For type 'search' and HTTPS URLs:
+ *   SEARCH_HTTPS_PROXY, SEARCH_HTTP_PROXY, search_https_proxy, search_http_proxy,
+ *   then HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For type 'search' and HTTP/unknown URLs:
+ *   SEARCH_HTTP_PROXY, SEARCH_HTTPS_PROXY, search_http_proxy, search_https_proxy,
+ *   then HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
+ * - For type 'url_reader' and HTTPS URLs:
+ *   URL_READER_HTTPS_PROXY, URL_READER_HTTP_PROXY, url_reader_https_proxy, url_reader_http_proxy,
+ *   then HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For type 'url_reader' and HTTP/unknown URLs:
+ *   URL_READER_HTTP_PROXY, URL_READER_HTTPS_PROXY, url_reader_http_proxy, url_reader_https_proxy,
+ *   then HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
+ * - For no specific type and HTTPS URLs:
+ *   HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For no specific type and HTTP/unknown URLs:
+ *   HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
  * - NO_PROXY / no_proxy: Comma-separated list of hosts to bypass proxy
  *
  * @param targetUrl - Optional target URL to check against NO_PROXY rules
+ * @param type - Optional proxy type ('search' or 'url_reader') for separate proxy configs
  * @returns ProxyAgent dispatcher for fetch, or undefined if no proxy configured or bypassed
  */
-export declare function createProxyAgent(targetUrl?: string): ProxyAgent | undefined;
+export declare function createProxyAgent(targetUrl?: string, type?: ProxyType): ProxyAgent | undefined;

package/dist/proxy.js

@@ -50,22 +50,116 @@ function shouldBypassProxy(targetUrl) {
     }
     return false;
 }
+/**
+ * Proxy configuration type for separating search and URL reader proxies.
+ */
+export const ProxyType = {
+    SEARCH: 'search',
+    URL_READER: 'url_reader',
+};
+/**
+ * Gets proxy URL for the specified proxy type.
+ * Checks type-specific proxy first, then falls back to global proxy.
+ *
+ * @param type - The type of proxy to get ('search' or 'url_reader')
+ * @param targetUrl - Optional target URL whose protocol is used to select between HTTP and HTTPS proxies
+ * @returns The proxy URL or undefined if not configured
+ */
+function getProxyUrl(type, targetUrl) {
+    let isHttps = false;
+    if (targetUrl) {
+        try {
+            const url = new URL(targetUrl);
+            isHttps = url.protocol === 'https:';
+        }
+        catch {
+            isHttps = false;
+        }
+    }
+    if (type === ProxyType.SEARCH) {
+        if (isHttps) {
+            return process.env.SEARCH_HTTPS_PROXY ||
+                process.env.SEARCH_HTTP_PROXY ||
+                process.env.search_https_proxy ||
+                process.env.search_http_proxy ||
+                process.env.HTTPS_PROXY ||
+                process.env.HTTP_PROXY ||
+                process.env.https_proxy ||
+                process.env.http_proxy;
+        }
+        return process.env.SEARCH_HTTP_PROXY ||
+            process.env.SEARCH_HTTPS_PROXY ||
+            process.env.search_http_proxy ||
+            process.env.search_https_proxy ||
+            // Fallback to global proxies
+            process.env.HTTP_PROXY ||
+            process.env.HTTPS_PROXY ||
+            process.env.http_proxy ||
+            process.env.https_proxy;
+    }
+    if (type === ProxyType.URL_READER) {
+        if (isHttps) {
+            return process.env.URL_READER_HTTPS_PROXY ||
+                process.env.URL_READER_HTTP_PROXY ||
+                process.env.url_reader_https_proxy ||
+                process.env.url_reader_http_proxy ||
+                process.env.HTTPS_PROXY ||
+                process.env.HTTP_PROXY ||
+                process.env.https_proxy ||
+                process.env.http_proxy;
+        }
+        return process.env.URL_READER_HTTP_PROXY ||
+            process.env.URL_READER_HTTPS_PROXY ||
+            process.env.url_reader_http_proxy ||
+            process.env.url_reader_https_proxy ||
+            // Fallback to global proxies
+            process.env.HTTP_PROXY ||
+            process.env.HTTPS_PROXY ||
+            process.env.http_proxy ||
+            process.env.https_proxy;
+    }
+    if (isHttps) {
+        return process.env.HTTPS_PROXY ||
+            process.env.HTTP_PROXY ||
+            process.env.https_proxy ||
+            process.env.http_proxy;
+    }
+    return process.env.HTTP_PROXY ||
+        process.env.HTTPS_PROXY ||
+        process.env.http_proxy ||
+        process.env.https_proxy;
+}
 /**
  * Creates a proxy agent dispatcher for Node.js fetch API.
  *
  * Node.js fetch uses Undici under the hood, which requires a 'dispatcher' option
  * instead of 'agent'. This function creates a ProxyAgent compatible with fetch.
  *
- * Environment variables checked (in order):
- * - HTTP_PROXY / http_proxy: For HTTP requests
- * - HTTPS_PROXY / https_proxy: For HTTPS requests
+ * Environment variables checked (in order, depending on URL protocol):
+ * - For type 'search' and HTTPS URLs:
+ *   SEARCH_HTTPS_PROXY, SEARCH_HTTP_PROXY, search_https_proxy, search_http_proxy,
+ *   then HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For type 'search' and HTTP/unknown URLs:
+ *   SEARCH_HTTP_PROXY, SEARCH_HTTPS_PROXY, search_http_proxy, search_https_proxy,
+ *   then HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
+ * - For type 'url_reader' and HTTPS URLs:
+ *   URL_READER_HTTPS_PROXY, URL_READER_HTTP_PROXY, url_reader_https_proxy, url_reader_http_proxy,
+ *   then HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For type 'url_reader' and HTTP/unknown URLs:
+ *   URL_READER_HTTP_PROXY, URL_READER_HTTPS_PROXY, url_reader_http_proxy, url_reader_https_proxy,
+ *   then HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
+ * - For no specific type and HTTPS URLs:
+ *   HTTPS_PROXY, HTTP_PROXY, https_proxy, http_proxy
+ * - For no specific type and HTTP/unknown URLs:
+ *   HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy
  * - NO_PROXY / no_proxy: Comma-separated list of hosts to bypass proxy
  *
  * @param targetUrl - Optional target URL to check against NO_PROXY rules
+ * @param type - Optional proxy type ('search' or 'url_reader') for separate proxy configs
  * @returns ProxyAgent dispatcher for fetch, or undefined if no proxy configured or bypassed
  */
-export function createProxyAgent(targetUrl) {
-    const proxyUrl = process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.http_proxy || process.env.https_proxy;
+export function createProxyAgent(targetUrl, type) {
+    const proxyUrl = getProxyUrl(type, targetUrl);
     if (!proxyUrl) {
         return undefined;
     }

package/dist/search.js

@@ -1,4 +1,4 @@
-import { createProxyAgent } from "./proxy.js";
+import { createProxyAgent, ProxyType } from "./proxy.js";
 import { logMessage } from "./logging.js";
 import { createConfigurationError, createNetworkError, createServerError, createJSONError, createDataError, createNoResultsMessage } from "./error-handler.js";
 export async function performWebSearch(server, query, pageno = 1, time_range, language = "all", safesearch) {
@@ -44,7 +44,7 @@ export async function performWebSearch(server, query, pageno = 1, time_range, la
     };
     // Add proxy dispatcher if proxy is configured
     // Node.js fetch uses 'dispatcher' option for proxy, not 'agent'
-    const proxyAgent = createProxyAgent(url.toString());
+    const proxyAgent = createProxyAgent(url.toString(), ProxyType.SEARCH);
     if (proxyAgent) {
         requestOptions.dispatcher = proxyAgent;
     }

package/dist/url-reader.js

@@ -1,5 +1,5 @@
 import { NodeHtmlMarkdown } from "node-html-markdown";
-import { createProxyAgent } from "./proxy.js";
+import { createProxyAgent, ProxyType } from "./proxy.js";
 import { logMessage } from "./logging.js";
 import { urlCache } from "./cache.js";
 import { createURLFormatError, createNetworkError, createServerError, createContentError, createConversionError, createTimeoutError, createEmptyContentWarning, createUnexpectedError } from "./error-handler.js";
@@ -131,10 +131,18 @@ export async function fetchAndConvertToMarkdown(server, url, timeoutMs = 10000,
         };
         // Add proxy dispatcher if proxy is configured
         // Node.js fetch uses 'dispatcher' option for proxy, not 'agent'
-        const proxyAgent = createProxyAgent(url);
+        const proxyAgent = createProxyAgent(url, ProxyType.URL_READER);
         if (proxyAgent) {
             requestOptions.dispatcher = proxyAgent;
         }
+        // Add User-Agent header if configured (URL_READER_USER_AGENT takes priority over USER_AGENT)
+        const userAgent = process.env.URL_READER_USER_AGENT || process.env.USER_AGENT;
+        if (userAgent) {
+            requestOptions.headers = {
+                ...requestOptions.headers,
+                'User-Agent': userAgent
+            };
+        }
         let response;
         try {
             // Fetch the URL with the abort signal

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcp-searxng",
-    "version": "0.9.2",
+    "version": "0.10.0",
     "description": "MCP server for SearXNG integration",
     "license": "MIT",
     "author": "Ihor Sokoliuk (https://github.com/ihor-sokoliuk)",
@@ -36,8 +36,8 @@
     "scripts": {
         "build": "tsc && shx chmod +x dist/*.js",
         "watch": "tsc --watch",
-        "test": "SEARXNG_URL=https://test-searx.example.com tsx __tests__/run-all.ts",
-        "test:coverage": "SEARXNG_URL=https://test-searx.example.com c8 --reporter=text tsx __tests__/run-all.ts",
+        "test": "cross-env SEARXNG_URL=https://test-searx.example.com tsx __tests__/run-all.ts",
+        "test:coverage": "cross-env SEARXNG_URL=https://test-searx.example.com c8 --reporter=text tsx __tests__/run-all.ts",
         "bootstrap": "npm install && npm run build",
         "inspector": "DANGEROUSLY_OMIT_AUTH=true npx @modelcontextprotocol/inspector node dist/index.js",
         "postversion": "TAG=$(node scripts/update-version.js | tail -1) && git add src/index.ts && git commit --amend --no-edit && git tag -f $TAG"
@@ -52,10 +52,11 @@
         "undici": "^6.20.1"
     },
     "devDependencies": {
-        "mcp-evals": "^1.0.18",
         "@types/node": "^22.17.2",
         "@types/supertest": "^6.0.3",
         "c8": "^10.1.3",
+        "cross-env": "^10.1.0",
+        "mcp-evals": "^1.0.18",
         "shx": "^0.4.0",
         "supertest": "^7.1.4",
         "tsx": "^4.20.5",