mcp-searxng 1.8.0 → 1.10.0

package/README.md

@@ -114,7 +114,13 @@ AI Assistant (e.g. Claude)
     - `refresh` (boolean, optional): Bypass the process cache and fetch fresh `/config` data. (default: false)
 
 - **web_url_read**
-  - Read and convert the content from a URL to markdown with advanced content extraction options
+  - Read URL content as markdown with content-type-aware handling and advanced extraction options
+  - Supported readable content:
+    - HTML (`text/html`, `application/xhtml+xml`) is converted to markdown
+    - JSON (`application/json`, `*+json`) is pretty-printed in a fenced block
+    - Plain text, YAML, TOML, XML, and other safe explicit `text/*` responses are returned as readable fenced text
+    - Missing or generic content types are read under the existing size cap; non-binary bodies continue through the HTML-to-markdown path for compatibility
+  - Binary, media, archive, PDF, and octet-stream downloads are intentionally rejected with a short hint instead of returning raw bytes
   - Inputs:
     - `url` (string): The URL to fetch and process
     - `startChar` (number, optional): Starting character position for content extraction (default: 0)
@@ -239,6 +245,8 @@ By default the server uses STDIO. Set `MCP_HTTP_PORT` to enable HTTP mode:
 
 **Endpoints:** `POST/GET/DELETE /mcp` (MCP protocol), `GET /health` (health check)
 
+For reverse-proxy deployments, see [CONFIGURATION.md](CONFIGURATION.md) for `MCP_HTTP_TRUST_PROXY` so rate limiting and logs use the correct client IP.
+
 **Test it:**
 
 ```bash
@@ -256,6 +264,8 @@ Full environment variable reference: [CONFIGURATION.md](CONFIGURATION.md)
 
 ## Troubleshooting
 
+If HTTPS requests fail behind a TLS-inspecting corporate proxy with certificate errors, see [TLS / Corporate CA](CONFIGURATION.md#tls--corporate-ca).
+
 ### 403 Forbidden from SearXNG
 
 Your SearXNG instance likely has JSON format disabled. Edit `settings.yml` (usually `/etc/searxng/settings.yml`):

package/dist/error-handler.d.ts

@@ -16,15 +16,15 @@ export declare class MCPSearXNGError extends Error {
 export declare function createConfigurationError(message: string): MCPSearXNGError;
 export declare function createNetworkError(error: any, context: ErrorContext): MCPSearXNGError;
 export declare function createServerError(status: number, statusText: string, responseBody: string, context: ErrorContext): MCPSearXNGError;
-export declare function createJSONError(responseText: string, context: ErrorContext): MCPSearXNGError;
-export declare function createDataError(data: any, context: ErrorContext): MCPSearXNGError;
+export declare function createJSONError(responseText: string): MCPSearXNGError;
+export declare function createDataError(): MCPSearXNGError;
 export declare function createNoResultsMessage(query: string): string;
 export declare function createURLFormatError(url: string): MCPSearXNGError;
 export declare function createURLSecurityPolicyError(url: string): MCPSearXNGError;
 export declare function createContentError(message: string, url: string): MCPSearXNGError;
-export declare function createConversionError(error: any, url: string, htmlContent: string): MCPSearXNGError;
+export declare function createConversionError(url: string): MCPSearXNGError;
 export declare function createTimeoutError(timeout: number, url: string): MCPSearXNGError;
-export declare function createEmptyContentWarning(url: string, htmlLength: number, htmlPreview: string): string;
+export declare function createEmptyContentWarning(url: string): string;
 export declare function createUnexpectedError(error: any, context: ErrorContext): MCPSearXNGError;
 /**
  * Process-level crash handlers, registered by the CLI entrypoint (cli.ts).

package/dist/error-handler.js

@@ -84,11 +84,11 @@ export function createServerError(status, statusText, responseBody, context) {
     }
     return new MCPSearXNGError(`🚫 ${target} Error (${status}): ${statusText}`);
 }
-export function createJSONError(responseText, context) {
+export function createJSONError(responseText) {
     const preview = responseText.substring(0, 100).replace(/\n/g, ' ');
-    return new MCPSearXNGError(`🔍 SearXNG Response Error: Invalid JSON format. Response: "${preview}..."`);
+    return new MCPSearXNGError(`🔍 SearXNG Response Error: Invalid JSON format. Response: "${preview}...". Enable - json under search.formats in your SearXNG settings.yml, or set SEARXNG_HTML_FALLBACK=true.`);
 }
-export function createDataError(data, context) {
+export function createDataError() {
     return new MCPSearXNGError(`🔍 SearXNG Data Error: Missing results array in response`);
 }
 export function createNoResultsMessage(query) {
@@ -104,14 +104,14 @@ export function createURLSecurityPolicyError(url) {
 export function createContentError(message, url) {
     return new MCPSearXNGError(`📄 Content Error: ${message} (${url})`);
 }
-export function createConversionError(error, url, htmlContent) {
+export function createConversionError(url) {
     return new MCPSearXNGError(`🔄 Conversion Error: Cannot convert HTML to Markdown (${url})`);
 }
 export function createTimeoutError(timeout, url) {
     const hostname = new URL(url).hostname;
     return new MCPSearXNGError(`⏱️ Timeout Error: ${hostname} took longer than ${timeout}ms to respond`);
 }
-export function createEmptyContentWarning(url, htmlLength, htmlPreview) {
+export function createEmptyContentWarning(url) {
     return `📄 Content Warning: Page fetched but appears empty after conversion (${url}). May contain only media or require JavaScript.`;
 }
 export function createUnexpectedError(error, context) {

package/dist/http-security.d.ts

@@ -6,6 +6,7 @@ export interface HttpSecurityConfig {
     allowedOrigins: string[];
     enableDnsRebindingProtection: boolean;
     allowedHosts: string[];
+    trustProxy: boolean | number | string;
     exposeFullConfig: boolean;
     allowPrivateUrls: boolean;
 }

package/dist/http-security.js

@@ -7,6 +7,21 @@ function parseCsv(value) {
         .map((item) => item.trim())
         .filter(Boolean);
 }
+function parseTrustProxy(value) {
+    const trimmed = value?.trim();
+    // Treat "0" as disabled (false): operators use 0 to turn numeric knobs off,
+    // and Express would otherwise mis-parse the string "0" as a bogus trust subnet.
+    if (!trimmed || trimmed === "false" || trimmed === "0") {
+        return false;
+    }
+    if (trimmed === "true") {
+        return true;
+    }
+    if (/^[1-9]\d*$/.test(trimmed)) {
+        return Number(trimmed);
+    }
+    return trimmed;
+}
 export function getHttpSecurityConfig() {
     const harden = isEnabled(process.env.MCP_HTTP_HARDEN);
     const authToken = process.env.MCP_HTTP_AUTH_TOKEN;
@@ -20,6 +35,7 @@ export function getHttpSecurityConfig() {
         allowedOrigins,
         enableDnsRebindingProtection: harden,
         allowedHosts: allowedHosts.length > 0 ? allowedHosts : ["127.0.0.1", "localhost"],
+        trustProxy: parseTrustProxy(process.env.MCP_HTTP_TRUST_PROXY),
         exposeFullConfig: isEnabled(process.env.MCP_HTTP_EXPOSE_FULL_CONFIG),
         allowPrivateUrls: isEnabled(process.env.MCP_HTTP_ALLOW_PRIVATE_URLS),
     };

package/dist/http-server.js

@@ -55,6 +55,9 @@ export async function createHttpServer(createMcpServer) {
     const app = express();
     const security = getHttpSecurityConfig();
     validateHttpSecurityConfig(security);
+    if (security.trustProxy !== false) {
+        app.set('trust proxy', security.trustProxy);
+    }
     app.use(express.json());
     // Add CORS support for web clients
     app.use(cors({
@@ -97,7 +100,7 @@ export async function createHttpServer(createMcpServer) {
             mcpServer = session.mcpServer;
             logMessage(mcpServer, "debug", `Reusing session: ${sessionId}`);
         }
-        else if (!sessionId && isInitializeRequest(req.body)) {
+        else if (isInitializeRequest(req.body)) {
             // New initialization request — create fresh McpServer and transport
             mcpServer = createMcpServer();
             transport = new StreamableHTTPServerTransport({
@@ -129,11 +132,12 @@ export async function createHttpServer(createMcpServer) {
                 contentType: req.headers['content-type'],
                 accept: req.headers['accept']
             });
-            res.status(400).json({
+            const sessionNotFound = Boolean(sessionId);
+            res.status(sessionNotFound ? 404 : 400).json({
                 jsonrpc: '2.0',
                 error: {
-                    code: -32000,
-                    message: 'Bad Request: No valid session ID provided',
+                    code: sessionNotFound ? -32001 : -32000,
+                    message: sessionNotFound ? 'Session not found' : 'Bad Request: No valid session ID provided',
                 },
                 id: null,
             });

package/dist/instance-info.js

@@ -1,26 +1,12 @@
 import { logMessage } from "./logging.js";
 import { createDefaultAgent, createProxyAgent, ProxyType } from "./proxy.js";
-import { getSearxngInstances } from "./searxng-instances.js";
+import { getSearxngInstances, redactSearxngInstanceUrl } from "./searxng-instances.js";
 const CONFIG_FAILURE_CACHE_TTL_MS = 60_000;
 const cachedConfigs = new Map();
 const cachedConfigFailures = new Map();
-function redactInstanceUrl(raw) {
-    try {
-        const url = new URL(raw);
-        if (!url.username && !url.password) {
-            return raw;
-        }
-        url.username = "";
-        url.password = "";
-        return url.toString();
-    }
-    catch {
-        return raw;
-    }
-}
 function redactFailures(failures) {
     return failures.map(({ sourceUrl, message, status }) => ({
-        sourceUrl: redactInstanceUrl(sourceUrl),
+        sourceUrl: redactSearxngInstanceUrl(sourceUrl),
         message,
         ...(status !== undefined ? { status } : {}),
     }));
@@ -45,30 +31,21 @@ function categoryNamesFromEngines(config) {
     }
     return [...names];
 }
-function categoryNamesFromArray(categories) {
+function categoryNamesFromList(values) {
     const names = new Set();
-    for (const category of categories) {
+    for (const category of values) {
         if (typeof category === "string" && category.trim() !== "") {
             names.add(category);
         }
     }
     return [...names];
 }
-function categoryNamesFromObject(categories) {
-    const names = new Set();
-    for (const category of Object.keys(categories)) {
-        if (category.trim() !== "") {
-            names.add(category);
-        }
-    }
-    return [...names];
-}
 function configuredCategoryNames(config) {
     if (Array.isArray(config.categories)) {
-        return categoryNamesFromArray(config.categories);
+        return categoryNamesFromList(config.categories);
     }
     if (config.categories && typeof config.categories === "object") {
-        return categoryNamesFromObject(config.categories);
+        return categoryNamesFromList(Object.keys(config.categories));
     }
     return [];
 }
@@ -180,7 +157,7 @@ function formatInstanceInfo(configs, failures, includeEngines, includeDisabled,
     const primary = configs[0].config;
     const payload = {
         available: true,
-        instancesReachable: configs.map(({ sourceUrl }) => redactInstanceUrl(sourceUrl)),
+        instancesReachable: configs.map(({ sourceUrl }) => redactSearxngInstanceUrl(sourceUrl)),
         ...(failures.length > 0 ? { instancesUnreachable: redactFailures(failures) } : {}),
         categories: aggregateCategories(configs, category),
         defaults: {
@@ -251,7 +228,9 @@ async function requestInstanceConfig(mcpServer, base) {
         return { available: true, config, sourceUrl: base };
     }
     catch (error) {
-        logMessage(mcpServer, "warning", `SearXNG /config fetch failed for ${redactInstanceUrl(base)}: ${error instanceof Error ? error.message : String(error)}`);
+        const rawMessage = error instanceof Error ? error.message : String(error);
+        const safeMessage = rawMessage.replaceAll(base, redactSearxngInstanceUrl(base));
+        logMessage(mcpServer, "warning", `SearXNG /config fetch failed for ${redactSearxngInstanceUrl(base)}: ${safeMessage}`);
         const message = "SearXNG /config is unavailable; instance capability discovery could not complete.";
         return {
             available: false,

package/dist/search.js

@@ -2,7 +2,7 @@ import { parse } from "node-html-parser";
 import { getKnownCategories, getKnownEngines } from "./instance-info.js";
 import { createProxyAgent, createDefaultAgent, ProxyType } from "./proxy.js";
 import { logMessage } from "./logging.js";
-import { getHealthySearxngInstances, getSearxngInstances, isSearxngFanoutEnabled, recordSearxngInstanceFailure, recordSearxngInstanceSuccess, } from "./searxng-instances.js";
+import { getHealthySearxngInstances, getSearxngInstances, isSearxngFanoutEnabled, recordSearxngInstanceFailure, recordSearxngInstanceSuccess, redactSearxngInstanceUrl, } from "./searxng-instances.js";
 import { MCPSearXNGError, validateEnvironment, createNetworkError, createServerError, createJSONError, createDataError, createNoResultsMessage } from "./error-handler.js";
 function getOperatorMaxResults(mcpServer) {
     const rawValue = process.env.SEARXNG_MAX_RESULTS;
@@ -88,22 +88,30 @@ function parseHtmlSearchResults(html, query) {
 async function fetchWithSearchTimeout(mcpServer, url, requestOptions, timeoutMs, query, searxngUrl) {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    const rawUrl = url.toString();
+    const redactedUrl = redactSearxngInstanceUrl(rawUrl);
     try {
-        logMessage(mcpServer, "info", `Making request to: ${url.toString()}`);
-        return await fetch(url.toString(), {
+        logMessage(mcpServer, "info", `Making request to: ${redactedUrl}`);
+        return await fetch(rawUrl, {
             ...requestOptions,
             signal: controller.signal,
         });
     }
     catch (error) {
-        logMessage(mcpServer, "error", `Network error during search request: ${error.message}`, { query, url: url.toString() });
+        const safeMessage = typeof error?.message === "string"
+            ? error.message.replaceAll(rawUrl, redactedUrl)
+            : error?.message;
+        const safeError = new Error(safeMessage);
+        safeError.code = error?.code;
+        safeError.cause = error?.cause;
+        logMessage(mcpServer, "error", `Network error during search request: ${safeMessage}`, { query, url: redactedUrl });
         const context = {
-            url: url.toString(),
-            searxngUrl,
+            url: redactedUrl,
+            searxngUrl: redactSearxngInstanceUrl(searxngUrl),
             proxyAgent: !!requestOptions.dispatcher,
             username: process.env.AUTH_USERNAME,
         };
-        throw createNetworkError(error, context);
+        throw createNetworkError(safeError, context);
     }
     finally {
         clearTimeout(timeoutId);
@@ -111,7 +119,7 @@ async function fetchWithSearchTimeout(mcpServer, url, requestOptions, timeoutMs,
 }
 async function fetchHtmlFallbackSearch(mcpServer, jsonUrl, requestOptions, timeoutMs, query, searxngUrl) {
     const htmlUrl = buildHtmlFallbackUrl(jsonUrl);
-    logMessage(mcpServer, "info", `Retrying search with HTML fallback: ${htmlUrl.toString()}`);
+    logMessage(mcpServer, "info", `Retrying search with HTML fallback: ${redactSearxngInstanceUrl(htmlUrl.toString())}`);
     const response = await fetchWithSearchTimeout(mcpServer, htmlUrl, requestOptions, timeoutMs, query, searxngUrl);
     if (!response.ok) {
         let responseBody;
@@ -122,8 +130,8 @@ async function fetchHtmlFallbackSearch(mcpServer, jsonUrl, requestOptions, timeo
             responseBody = '[Could not read response body]';
         }
         const context = {
-            url: htmlUrl.toString(),
-            searxngUrl,
+            url: redactSearxngInstanceUrl(htmlUrl.toString()),
+            searxngUrl: redactSearxngInstanceUrl(searxngUrl),
         };
         throw createServerError(response.status, response.statusText, responseBody, context);
     }
@@ -330,36 +338,37 @@ async function fetchSearchFromInstance(mcpServer, instanceUrl, request) {
                 responseBody = '[Could not read response body]';
             }
             const context = {
-                url: url.toString(),
-                searxngUrl: instanceUrl
+                url: redactSearxngInstanceUrl(url.toString()),
+                searxngUrl: redactSearxngInstanceUrl(instanceUrl)
             };
             throw createServerError(response.status, response.statusText, responseBody, context);
         }
     }
     else {
+        // Read the body as text once, then parse — a Response body is single-use, so
+        // calling response.text() after response.json() consumed it always throws and
+        // would leave the error preview empty.
+        let responseText;
         try {
-            data = (await response.json());
+            responseText = await response.text();
         }
-        catch (error) {
+        catch {
+            responseText = '[Could not read response text]';
+        }
+        try {
+            data = JSON.parse(responseText);
+        }
+        catch {
             if (isHtmlFallbackEnabled()) {
                 data = await fetchHtmlFallbackSearch(mcpServer, url, requestOptions, request.timeoutMs, request.query, instanceUrl);
             }
             else {
-                let responseText;
-                try {
-                    responseText = await response.text();
-                }
-                catch {
-                    responseText = '[Could not read response text]';
-                }
-                const context = { url: url.toString() };
-                throw createJSONError(responseText, context);
+                throw createJSONError(responseText);
             }
         }
     }
     if (!data.results) {
-        const context = { url: url.toString(), query: request.query };
-        throw createDataError(data, context);
+        throw createDataError();
     }
     return {
         instanceUrl,
@@ -371,13 +380,13 @@ function hasSearchResults(data) {
 }
 function createAllInstancesFailedError(failures, skippedInstances) {
     if (failures.length === 0 && skippedInstances.length > 0) {
-        return new MCPSearXNGError(`All configured SearXNG instances are in cooldown after repeated failures: ${skippedInstances.join(", ")}.`);
+        return new MCPSearXNGError(`All configured SearXNG instances are in cooldown after repeated failures: ${skippedInstances.map(redactSearxngInstanceUrl).join(", ")}.`);
     }
     const failureDetails = failures
-        .map(({ instanceUrl, error }) => `${instanceUrl}: ${error instanceof Error ? error.message : String(error)}`)
+        .map(({ instanceUrl, error }) => `${redactSearxngInstanceUrl(instanceUrl)}: ${error instanceof Error ? error.message : String(error)}`)
         .join("; ");
     const skippedDetails = skippedInstances.length > 0
-        ? ` Skipped cooled-down instances: ${skippedInstances.join(", ")}.`
+        ? ` Skipped cooled-down instances: ${skippedInstances.map(redactSearxngInstanceUrl).join(", ")}.`
         : "";
     return new MCPSearXNGError(`All configured SearXNG instances failed. ${failureDetails}${skippedDetails}`);
 }
@@ -535,6 +544,7 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
         data = multiResult.data;
         servedBy = multiResult.servedBy;
     }
+    const redactedServedBy = servedBy.map(redactSearxngInstanceUrl);
     const results = data.results
         .filter((result) => min_score === undefined || (result.score || 0) >= min_score);
     const slicedResults = effectiveMax !== undefined
@@ -545,13 +555,13 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
             ...data,
             results: slicedResults,
             ...(filters.validationWarning ? { warnings: [filters.validationWarning] } : {}),
-            ...(includeProvenance ? { servedBy } : {}),
+            ...(includeProvenance ? { servedBy: redactedServedBy } : {}),
         }, null, 2);
     }
     const metadata = formatSearchMetadata(data);
     const leadingSections = [
         includeProvenance
-            ? `Served by SearXNG ${servedBy.length === 1 ? "instance" : "instances"}: ${servedBy.join(", ")}`
+            ? `Served by SearXNG ${redactedServedBy.length === 1 ? "instance" : "instances"}: ${redactedServedBy.join(", ")}`
             : null,
         filters.validationNote ?? null,
         data.sourceFormat === "html" ? "Note: Results parsed from SearXNG HTML fallback; metadata is limited." : null,

package/dist/searxng-instances.d.ts

@@ -2,6 +2,7 @@ export declare function parseSearxngUrls(raw?: string | undefined): string[];
 export declare function getSearxngInstances(): string[];
 export declare function getPrimarySearxngInstance(): string | undefined;
 export declare function validateSearxngInstanceUrl(value: string): string | null;
+export declare function redactSearxngInstanceUrl(raw: string): string;
 export declare function isSearxngFanoutEnabled(): boolean;
 export declare function recordSearxngInstanceFailure(instanceUrl: string, now?: number): void;
 export declare function recordSearxngInstanceSuccess(instanceUrl: string): void;

package/dist/searxng-instances.js

@@ -39,6 +39,20 @@ export function validateSearxngInstanceUrl(value) {
     }
     return null;
 }
+export function redactSearxngInstanceUrl(raw) {
+    try {
+        const url = new URL(raw);
+        if (!url.username && !url.password) {
+            return raw;
+        }
+        url.username = "";
+        url.password = "";
+        return url.toString();
+    }
+    catch {
+        return raw.replace(/^([a-zA-Z][a-zA-Z0-9+.-]*:\/\/)[^/]*@/, "$1");
+    }
+}
 export function isSearxngFanoutEnabled() {
     return process.env.SEARXNG_FANOUT === "true";
 }

package/dist/types.js

@@ -237,7 +237,7 @@ export const LITE_INSTANCE_INFO_TOOL = {
 };
 export const LITE_READ_URL_TOOL = {
     name: "web_url_read",
-    description: "Fetch URL. Returns page text as markdown.",
+    description: "Fetch URL. Converts HTML to markdown; returns explicit JSON, plain text, YAML, TOML, and XML as readable markdown; binary/media/archive downloads are rejected.",
     inputSchema: {
         type: "object",
         properties: { url: { type: "string", description: "URL to fetch." } },
@@ -246,7 +246,9 @@ export const LITE_READ_URL_TOOL = {
 };
 export const READ_URL_TOOL = {
     name: "web_url_read",
-    description: "Fetches a URL and returns its text content converted to markdown. " +
+    description: "Fetches a URL and returns readable content as markdown. " +
+        "Content-type aware: HTML is converted to markdown; JSON is pretty-printed; plain text, YAML, TOML, and XML are returned as fenced readable text. " +
+        "Binary, media, archive, PDF, and octet-stream downloads are intentionally rejected instead of being returned as raw bytes. " +
         "Three modes: " +
         "(1) Full content — omit filtering params; use `startChar`/`maxLength` to paginate large pages. " +
         "(2) Section extraction — set `section` to return content under a specific heading. " +

package/dist/url-reader.js

@@ -9,6 +9,40 @@ const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
 const MAX_REDIRECTS = 5;
 export const DEFAULT_MAX_CONTENT_LENGTH_BYTES = 5 * 1024 * 1024;
 const HEAD_TIMEOUT_CAP_MS = 3000;
+const BINARY_SNIFF_PREFIX_BYTES = 1024;
+const EXACT_READABLE_CONTENT_TYPES = new Map([
+    ["text/html", (mediaType) => ({ kind: "html", mediaType, language: "html" })],
+    ["application/xhtml+xml", (mediaType) => ({ kind: "html", mediaType, language: "html" })],
+    ["application/json", (mediaType) => ({ kind: "json", mediaType, language: "json" })],
+    ["application/xml", (mediaType) => ({ kind: "text", mediaType, language: "xml" })],
+    ["text/xml", (mediaType) => ({ kind: "text", mediaType, language: "xml" })],
+    ["application/yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["application/x-yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["text/yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["text/x-yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["application/toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+    ["application/x-toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+    ["text/toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+]);
+const EXACT_BINARY_CONTENT_TYPES = new Set([
+    "application/pdf",
+    "application/octet-stream",
+    "binary/octet-stream",
+    "application/zip",
+    "application/x-zip",
+    "application/x-zip-compressed",
+    "application/gzip",
+    "application/x-gzip",
+    "application/x-tar",
+    "application/tar",
+    "application/x-7z-compressed",
+    "application/x-rar-compressed",
+    "application/vnd.rar",
+    "application/x-bzip",
+    "application/x-bzip2",
+    "application/x-xz",
+    "application/zstd",
+]);
 function isRedirectResponse(response) {
     return REDIRECT_STATUS_CODES.has(response.status);
 }
@@ -160,6 +194,93 @@ function createContentTooLargeMessage(contentLength, maxBytes) {
     return (`Content too large: server reports Content-Length of ${sizeMB} MB (limit: ${limitMB} MB). ` +
         `Try using readHeadings or section to fetch only the relevant parts.`);
 }
+function normalizeMediaType(contentType) {
+    if (!contentType) {
+        return null;
+    }
+    const mediaType = contentType.split(";")[0].trim().toLowerCase();
+    return mediaType === "" ? null : mediaType;
+}
+function isBinaryMediaType(mediaType) {
+    if (mediaType.startsWith("image/") ||
+        mediaType.startsWith("audio/") ||
+        mediaType.startsWith("video/") ||
+        mediaType.startsWith("font/")) {
+        return true;
+    }
+    return EXACT_BINARY_CONTENT_TYPES.has(mediaType);
+}
+function classifyContentType(contentType) {
+    const mediaType = normalizeMediaType(contentType);
+    if (mediaType === null) {
+        return { kind: "generic", mediaType };
+    }
+    const exactReadable = EXACT_READABLE_CONTENT_TYPES.get(mediaType);
+    if (exactReadable) {
+        return exactReadable(mediaType);
+    }
+    if (mediaType.endsWith("+json")) {
+        return { kind: "json", mediaType, language: "json" };
+    }
+    else if (isBinaryMediaType(mediaType)) {
+        return { kind: "binary", mediaType };
+    }
+    else if (mediaType.endsWith("+xml")) {
+        return { kind: "text", mediaType, language: "xml" };
+    }
+    else if (mediaType.startsWith("text/")) {
+        return { kind: "text", mediaType, language: "text" };
+    }
+    return { kind: "generic", mediaType };
+}
+function createUnsupportedContentTypeMessage(classification, reason) {
+    const contentType = classification.mediaType ?? "missing";
+    const reasonText = reason ? ` ${reason}` : "";
+    return (`Unsupported content type: ${contentType}.${reasonText} ` +
+        "Binary, media, archive, and PDF downloads are intentionally not read by web_url_read.");
+}
+function createNulRejectedContentMessage(classification) {
+    if (classification.kind !== "generic" && classification.mediaType !== null) {
+        return (`Body was declared ${classification.mediaType} but appears binary (NUL byte in first 1KB); not read. ` +
+            "Binary, media, archive, and PDF downloads are intentionally not read by web_url_read.");
+    }
+    return createUnsupportedContentTypeMessage(classification, `Body appears binary: NUL byte found in the first ${BINARY_SNIFF_PREFIX_BYTES} bytes.`);
+}
+async function cancelResponseBody(response) {
+    try {
+        await response.body?.cancel();
+    }
+    catch {
+        // Best-effort cancellation: returning the unsupported hint is more useful than surfacing cancellation noise.
+    }
+}
+function getLongestBacktickRun(text) {
+    let longestRun = 0;
+    let currentRun = 0;
+    for (const char of text) {
+        if (char === "`") {
+            currentRun++;
+            longestRun = Math.max(longestRun, currentRun);
+        }
+        else {
+            currentRun = 0;
+        }
+    }
+    return longestRun;
+}
+function renderFencedMarkdown(language, text) {
+    const fence = "`".repeat(Math.max(3, getLongestBacktickRun(text) + 1));
+    return `${fence}${language}\n${text}\n${fence}`;
+}
+function renderJsonMarkdown(text) {
+    try {
+        const parsed = JSON.parse(text);
+        return renderFencedMarkdown("json", JSON.stringify(parsed, null, 2));
+    }
+    catch {
+        return `Note: Response declared JSON but could not be parsed.\n\n${renderFencedMarkdown("text", text)}`;
+    }
+}
 function concatenateChunks(chunks, totalBytes) {
     const result = new Uint8Array(totalBytes);
     let offset = 0;
@@ -169,13 +290,35 @@ function concatenateChunks(chunks, totalBytes) {
     }
     return result;
 }
-async function readResponseBodyWithLimit(response, maxBytes) {
+function scanPrefixForNul(value, prefixBytesChecked) {
+    const remainingPrefixBytes = BINARY_SNIFF_PREFIX_BYTES - prefixBytesChecked;
+    const bytesToCheck = Math.min(value.byteLength, remainingPrefixBytes);
+    if (bytesToCheck <= 0) {
+        return { hasNul: false, prefixBytesChecked };
+    }
+    return {
+        hasNul: value.subarray(0, bytesToCheck).includes(0),
+        prefixBytesChecked: prefixBytesChecked + bytesToCheck,
+    };
+}
+function evaluateChunkLimits(bytesRead, maxBytes, hasNulInPrefix, abortOnNulInPrefix) {
+    if (hasNulInPrefix && abortOnNulInPrefix) {
+        return { exceeded: false, text: "", bytesRead, hasNulInPrefix };
+    }
+    if (bytesRead > maxBytes) {
+        return { exceeded: true, bytesRead };
+    }
+    return null;
+}
+async function readResponseBodyWithLimit(response, maxBytes, abortOnNulInPrefix = false) {
     if (response.body === null) {
-        return { exceeded: false, text: "", bytesRead: 0 };
+        return { exceeded: false, text: "", bytesRead: 0, hasNulInPrefix: false };
     }
     const reader = response.body.getReader();
     const chunks = [];
     let bytesRead = 0;
+    let prefixBytesChecked = 0;
+    let hasNulInPrefix = false;
     try {
         while (true) {
             const { done, value } = await reader.read();
@@ -185,10 +328,14 @@ async function readResponseBodyWithLimit(response, maxBytes) {
             if (!value) {
                 continue;
             }
+            const nulScan = scanPrefixForNul(value, prefixBytesChecked);
+            hasNulInPrefix = hasNulInPrefix || nulScan.hasNul;
+            prefixBytesChecked = nulScan.prefixBytesChecked;
             bytesRead += value.byteLength;
-            if (bytesRead > maxBytes) {
+            const limitResult = evaluateChunkLimits(bytesRead, maxBytes, hasNulInPrefix, abortOnNulInPrefix);
+            if (limitResult) {
                 await reader.cancel();
-                return { exceeded: true, bytesRead };
+                return limitResult;
             }
             chunks.push(value);
         }
@@ -197,7 +344,7 @@ async function readResponseBodyWithLimit(response, maxBytes) {
         reader.releaseLock();
     }
     const bodyBytes = concatenateChunks(chunks, bytesRead);
-    return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead };
+    return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead, hasNulInPrefix };
 }
 export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 10000, paginationOptions = {}) {
     const startTime = Date.now();
@@ -306,36 +453,54 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
             const context = { url };
             throw createServerError(response.status, response.statusText, responseBody, context);
         }
-        // Retrieve HTML content
-        let htmlContent;
+        const contentType = classifyContentType(response.headers.get("content-type"));
+        if (contentType.kind === "binary") {
+            await cancelResponseBody(response);
+            return createUnsupportedContentTypeMessage(contentType);
+        }
+        // Retrieve readable content
+        let rawContent;
+        let hasNulInPrefix = false;
         try {
-            const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+            const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes, true);
             if (bodyRead.exceeded) {
                 return createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes);
             }
-            htmlContent = bodyRead.text;
+            rawContent = bodyRead.text;
+            hasNulInPrefix = bodyRead.hasNulInPrefix;
         }
         catch (error) {
             throw createContentError(`Failed to read website content: ${error.message || 'Unknown error reading content'}`, url);
         }
-        if (!htmlContent || htmlContent.trim().length === 0) {
+        if (hasNulInPrefix) {
+            return createNulRejectedContentMessage(contentType);
+        }
+        if (!rawContent || rawContent.trim().length === 0) {
             throw createContentError("Website returned empty content.", url);
         }
-        // Convert HTML to Markdown
+        // Convert readable content to Markdown
         let markdownContent;
-        try {
-            markdownContent = NodeHtmlMarkdown.translate(htmlContent);
+        if (contentType.kind === "json") {
+            markdownContent = renderJsonMarkdown(rawContent);
         }
-        catch (error) {
-            throw createConversionError(error, url, htmlContent);
+        else if (contentType.kind === "text") {
+            markdownContent = renderFencedMarkdown(contentType.language, rawContent);
+        }
+        else {
+            try {
+                markdownContent = NodeHtmlMarkdown.translate(rawContent);
+            }
+            catch {
+                throw createConversionError(url);
+            }
         }
         if (!markdownContent || markdownContent.trim().length === 0) {
             logMessage(mcpServer, "warning", `Empty content after conversion: ${url}`);
             // DON'T cache empty/failed conversions - return warning directly
-            return createEmptyContentWarning(url, htmlContent.length, htmlContent);
+            return createEmptyContentWarning(url);
         }
         // Only cache successful markdown conversion
-        urlCache.set(url, htmlContent, markdownContent);
+        urlCache.set(url, rawContent, markdownContent);
         // Apply pagination options
         const result = applyPaginationOptions(markdownContent, paginationOptions);
         const duration = Date.now() - startTime;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const packageVersion = "1.8.0";
+export declare const packageVersion = "1.10.0";

package/dist/version.js

@@ -1 +1 @@
-export const packageVersion = "1.8.0";
+export const packageVersion = "1.10.0";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcp-searxng",
-    "version": "1.8.0",
+    "version": "1.10.0",
     "mcpName": "io.github.ihor-sokoliuk/mcp-searxng",
     "description": "MCP server for SearXNG integration",
     "license": "MIT",
@@ -61,11 +61,11 @@
     "devDependencies": {
         "@types/node": "22.20.0",
         "@types/supertest": "^7.2.0",
-        "@typescript-eslint/eslint-plugin": "8.61.1",
-        "@typescript-eslint/parser": "8.61.1",
+        "@typescript-eslint/eslint-plugin": "8.62.0",
+        "@typescript-eslint/parser": "8.62.0",
         "c8": "^11.0.0",
         "cross-env": "^10.1.0",
-        "eslint": "10.5.0",
+        "eslint": "10.6.0",
         "eslint-plugin-security": "^4.0.0",
         "fast-check": "^4.8.0",
         "shx": "^0.4.0",