mcp-searxng 1.9.0 → 1.10.0

package/README.md

@@ -114,7 +114,13 @@ AI Assistant (e.g. Claude)
     - `refresh` (boolean, optional): Bypass the process cache and fetch fresh `/config` data. (default: false)
 
 - **web_url_read**
-  - Read and convert the content from a URL to markdown with advanced content extraction options
+  - Read URL content as markdown with content-type-aware handling and advanced extraction options
+  - Supported readable content:
+    - HTML (`text/html`, `application/xhtml+xml`) is converted to markdown
+    - JSON (`application/json`, `*+json`) is pretty-printed in a fenced block
+    - Plain text, YAML, TOML, XML, and other safe explicit `text/*` responses are returned as readable fenced text
+    - Missing or generic content types are read under the existing size cap; non-binary bodies continue through the HTML-to-markdown path for compatibility
+  - Binary, media, archive, PDF, and octet-stream downloads are intentionally rejected with a short hint instead of returning raw bytes
   - Inputs:
     - `url` (string): The URL to fetch and process
     - `startChar` (number, optional): Starting character position for content extraction (default: 0)
@@ -258,6 +264,8 @@ Full environment variable reference: [CONFIGURATION.md](CONFIGURATION.md)
 
 ## Troubleshooting
 
+If HTTPS requests fail behind a TLS-inspecting corporate proxy with certificate errors, see [TLS / Corporate CA](CONFIGURATION.md#tls--corporate-ca).
+
 ### 403 Forbidden from SearXNG
 
 Your SearXNG instance likely has JSON format disabled. Edit `settings.yml` (usually `/etc/searxng/settings.yml`):

package/dist/error-handler.js

@@ -86,7 +86,7 @@ export function createServerError(status, statusText, responseBody, context) {
 }
 export function createJSONError(responseText) {
     const preview = responseText.substring(0, 100).replace(/\n/g, ' ');
-    return new MCPSearXNGError(`🔍 SearXNG Response Error: Invalid JSON format. Response: "${preview}..."`);
+    return new MCPSearXNGError(`🔍 SearXNG Response Error: Invalid JSON format. Response: "${preview}...". Enable - json under search.formats in your SearXNG settings.yml, or set SEARXNG_HTML_FALLBACK=true.`);
 }
 export function createDataError() {
     return new MCPSearXNGError(`🔍 SearXNG Data Error: Missing results array in response`);

package/dist/types.js

@@ -237,7 +237,7 @@ export const LITE_INSTANCE_INFO_TOOL = {
 };
 export const LITE_READ_URL_TOOL = {
     name: "web_url_read",
-    description: "Fetch URL. Returns page text as markdown.",
+    description: "Fetch URL. Converts HTML to markdown; returns explicit JSON, plain text, YAML, TOML, and XML as readable markdown; binary/media/archive downloads are rejected.",
     inputSchema: {
         type: "object",
         properties: { url: { type: "string", description: "URL to fetch." } },
@@ -246,7 +246,9 @@ export const LITE_READ_URL_TOOL = {
 };
 export const READ_URL_TOOL = {
     name: "web_url_read",
-    description: "Fetches a URL and returns its text content converted to markdown. " +
+    description: "Fetches a URL and returns readable content as markdown. " +
+        "Content-type aware: HTML is converted to markdown; JSON is pretty-printed; plain text, YAML, TOML, and XML are returned as fenced readable text. " +
+        "Binary, media, archive, PDF, and octet-stream downloads are intentionally rejected instead of being returned as raw bytes. " +
         "Three modes: " +
         "(1) Full content — omit filtering params; use `startChar`/`maxLength` to paginate large pages. " +
         "(2) Section extraction — set `section` to return content under a specific heading. " +

package/dist/url-reader.js

@@ -9,6 +9,40 @@ const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
 const MAX_REDIRECTS = 5;
 export const DEFAULT_MAX_CONTENT_LENGTH_BYTES = 5 * 1024 * 1024;
 const HEAD_TIMEOUT_CAP_MS = 3000;
+const BINARY_SNIFF_PREFIX_BYTES = 1024;
+const EXACT_READABLE_CONTENT_TYPES = new Map([
+    ["text/html", (mediaType) => ({ kind: "html", mediaType, language: "html" })],
+    ["application/xhtml+xml", (mediaType) => ({ kind: "html", mediaType, language: "html" })],
+    ["application/json", (mediaType) => ({ kind: "json", mediaType, language: "json" })],
+    ["application/xml", (mediaType) => ({ kind: "text", mediaType, language: "xml" })],
+    ["text/xml", (mediaType) => ({ kind: "text", mediaType, language: "xml" })],
+    ["application/yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["application/x-yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["text/yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["text/x-yaml", (mediaType) => ({ kind: "text", mediaType, language: "yaml" })],
+    ["application/toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+    ["application/x-toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+    ["text/toml", (mediaType) => ({ kind: "text", mediaType, language: "toml" })],
+]);
+const EXACT_BINARY_CONTENT_TYPES = new Set([
+    "application/pdf",
+    "application/octet-stream",
+    "binary/octet-stream",
+    "application/zip",
+    "application/x-zip",
+    "application/x-zip-compressed",
+    "application/gzip",
+    "application/x-gzip",
+    "application/x-tar",
+    "application/tar",
+    "application/x-7z-compressed",
+    "application/x-rar-compressed",
+    "application/vnd.rar",
+    "application/x-bzip",
+    "application/x-bzip2",
+    "application/x-xz",
+    "application/zstd",
+]);
 function isRedirectResponse(response) {
     return REDIRECT_STATUS_CODES.has(response.status);
 }
@@ -160,6 +194,93 @@ function createContentTooLargeMessage(contentLength, maxBytes) {
     return (`Content too large: server reports Content-Length of ${sizeMB} MB (limit: ${limitMB} MB). ` +
         `Try using readHeadings or section to fetch only the relevant parts.`);
 }
+function normalizeMediaType(contentType) {
+    if (!contentType) {
+        return null;
+    }
+    const mediaType = contentType.split(";")[0].trim().toLowerCase();
+    return mediaType === "" ? null : mediaType;
+}
+function isBinaryMediaType(mediaType) {
+    if (mediaType.startsWith("image/") ||
+        mediaType.startsWith("audio/") ||
+        mediaType.startsWith("video/") ||
+        mediaType.startsWith("font/")) {
+        return true;
+    }
+    return EXACT_BINARY_CONTENT_TYPES.has(mediaType);
+}
+function classifyContentType(contentType) {
+    const mediaType = normalizeMediaType(contentType);
+    if (mediaType === null) {
+        return { kind: "generic", mediaType };
+    }
+    const exactReadable = EXACT_READABLE_CONTENT_TYPES.get(mediaType);
+    if (exactReadable) {
+        return exactReadable(mediaType);
+    }
+    if (mediaType.endsWith("+json")) {
+        return { kind: "json", mediaType, language: "json" };
+    }
+    else if (isBinaryMediaType(mediaType)) {
+        return { kind: "binary", mediaType };
+    }
+    else if (mediaType.endsWith("+xml")) {
+        return { kind: "text", mediaType, language: "xml" };
+    }
+    else if (mediaType.startsWith("text/")) {
+        return { kind: "text", mediaType, language: "text" };
+    }
+    return { kind: "generic", mediaType };
+}
+function createUnsupportedContentTypeMessage(classification, reason) {
+    const contentType = classification.mediaType ?? "missing";
+    const reasonText = reason ? ` ${reason}` : "";
+    return (`Unsupported content type: ${contentType}.${reasonText} ` +
+        "Binary, media, archive, and PDF downloads are intentionally not read by web_url_read.");
+}
+function createNulRejectedContentMessage(classification) {
+    if (classification.kind !== "generic" && classification.mediaType !== null) {
+        return (`Body was declared ${classification.mediaType} but appears binary (NUL byte in first 1KB); not read. ` +
+            "Binary, media, archive, and PDF downloads are intentionally not read by web_url_read.");
+    }
+    return createUnsupportedContentTypeMessage(classification, `Body appears binary: NUL byte found in the first ${BINARY_SNIFF_PREFIX_BYTES} bytes.`);
+}
+async function cancelResponseBody(response) {
+    try {
+        await response.body?.cancel();
+    }
+    catch {
+        // Best-effort cancellation: returning the unsupported hint is more useful than surfacing cancellation noise.
+    }
+}
+function getLongestBacktickRun(text) {
+    let longestRun = 0;
+    let currentRun = 0;
+    for (const char of text) {
+        if (char === "`") {
+            currentRun++;
+            longestRun = Math.max(longestRun, currentRun);
+        }
+        else {
+            currentRun = 0;
+        }
+    }
+    return longestRun;
+}
+function renderFencedMarkdown(language, text) {
+    const fence = "`".repeat(Math.max(3, getLongestBacktickRun(text) + 1));
+    return `${fence}${language}\n${text}\n${fence}`;
+}
+function renderJsonMarkdown(text) {
+    try {
+        const parsed = JSON.parse(text);
+        return renderFencedMarkdown("json", JSON.stringify(parsed, null, 2));
+    }
+    catch {
+        return `Note: Response declared JSON but could not be parsed.\n\n${renderFencedMarkdown("text", text)}`;
+    }
+}
 function concatenateChunks(chunks, totalBytes) {
     const result = new Uint8Array(totalBytes);
     let offset = 0;
@@ -169,13 +290,35 @@ function concatenateChunks(chunks, totalBytes) {
     }
     return result;
 }
-async function readResponseBodyWithLimit(response, maxBytes) {
+function scanPrefixForNul(value, prefixBytesChecked) {
+    const remainingPrefixBytes = BINARY_SNIFF_PREFIX_BYTES - prefixBytesChecked;
+    const bytesToCheck = Math.min(value.byteLength, remainingPrefixBytes);
+    if (bytesToCheck <= 0) {
+        return { hasNul: false, prefixBytesChecked };
+    }
+    return {
+        hasNul: value.subarray(0, bytesToCheck).includes(0),
+        prefixBytesChecked: prefixBytesChecked + bytesToCheck,
+    };
+}
+function evaluateChunkLimits(bytesRead, maxBytes, hasNulInPrefix, abortOnNulInPrefix) {
+    if (hasNulInPrefix && abortOnNulInPrefix) {
+        return { exceeded: false, text: "", bytesRead, hasNulInPrefix };
+    }
+    if (bytesRead > maxBytes) {
+        return { exceeded: true, bytesRead };
+    }
+    return null;
+}
+async function readResponseBodyWithLimit(response, maxBytes, abortOnNulInPrefix = false) {
     if (response.body === null) {
-        return { exceeded: false, text: "", bytesRead: 0 };
+        return { exceeded: false, text: "", bytesRead: 0, hasNulInPrefix: false };
     }
     const reader = response.body.getReader();
     const chunks = [];
     let bytesRead = 0;
+    let prefixBytesChecked = 0;
+    let hasNulInPrefix = false;
     try {
         while (true) {
             const { done, value } = await reader.read();
@@ -185,10 +328,14 @@ async function readResponseBodyWithLimit(response, maxBytes) {
             if (!value) {
                 continue;
             }
+            const nulScan = scanPrefixForNul(value, prefixBytesChecked);
+            hasNulInPrefix = hasNulInPrefix || nulScan.hasNul;
+            prefixBytesChecked = nulScan.prefixBytesChecked;
             bytesRead += value.byteLength;
-            if (bytesRead > maxBytes) {
+            const limitResult = evaluateChunkLimits(bytesRead, maxBytes, hasNulInPrefix, abortOnNulInPrefix);
+            if (limitResult) {
                 await reader.cancel();
-                return { exceeded: true, bytesRead };
+                return limitResult;
             }
             chunks.push(value);
         }
@@ -197,7 +344,7 @@ async function readResponseBodyWithLimit(response, maxBytes) {
         reader.releaseLock();
     }
     const bodyBytes = concatenateChunks(chunks, bytesRead);
-    return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead };
+    return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead, hasNulInPrefix };
 }
 export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 10000, paginationOptions = {}) {
     const startTime = Date.now();
@@ -306,28 +453,46 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
             const context = { url };
             throw createServerError(response.status, response.statusText, responseBody, context);
         }
-        // Retrieve HTML content
-        let htmlContent;
+        const contentType = classifyContentType(response.headers.get("content-type"));
+        if (contentType.kind === "binary") {
+            await cancelResponseBody(response);
+            return createUnsupportedContentTypeMessage(contentType);
+        }
+        // Retrieve readable content
+        let rawContent;
+        let hasNulInPrefix = false;
         try {
-            const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+            const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes, true);
             if (bodyRead.exceeded) {
                 return createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes);
             }
-            htmlContent = bodyRead.text;
+            rawContent = bodyRead.text;
+            hasNulInPrefix = bodyRead.hasNulInPrefix;
         }
         catch (error) {
             throw createContentError(`Failed to read website content: ${error.message || 'Unknown error reading content'}`, url);
         }
-        if (!htmlContent || htmlContent.trim().length === 0) {
+        if (hasNulInPrefix) {
+            return createNulRejectedContentMessage(contentType);
+        }
+        if (!rawContent || rawContent.trim().length === 0) {
             throw createContentError("Website returned empty content.", url);
         }
-        // Convert HTML to Markdown
+        // Convert readable content to Markdown
         let markdownContent;
-        try {
-            markdownContent = NodeHtmlMarkdown.translate(htmlContent);
+        if (contentType.kind === "json") {
+            markdownContent = renderJsonMarkdown(rawContent);
         }
-        catch {
-            throw createConversionError(url);
+        else if (contentType.kind === "text") {
+            markdownContent = renderFencedMarkdown(contentType.language, rawContent);
+        }
+        else {
+            try {
+                markdownContent = NodeHtmlMarkdown.translate(rawContent);
+            }
+            catch {
+                throw createConversionError(url);
+            }
         }
         if (!markdownContent || markdownContent.trim().length === 0) {
             logMessage(mcpServer, "warning", `Empty content after conversion: ${url}`);
@@ -335,7 +500,7 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
             return createEmptyContentWarning(url);
         }
         // Only cache successful markdown conversion
-        urlCache.set(url, htmlContent, markdownContent);
+        urlCache.set(url, rawContent, markdownContent);
         // Apply pagination options
         const result = applyPaginationOptions(markdownContent, paginationOptions);
         const duration = Date.now() - startTime;

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const packageVersion = "1.9.0";
+export declare const packageVersion = "1.10.0";

package/dist/version.js

@@ -1 +1 @@
-export const packageVersion = "1.9.0";
+export const packageVersion = "1.10.0";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcp-searxng",
-    "version": "1.9.0",
+    "version": "1.10.0",
     "mcpName": "io.github.ihor-sokoliuk/mcp-searxng",
     "description": "MCP server for SearXNG integration",
     "license": "MIT",