mcp-searxng 1.6.0 → 1.7.0

package/README.md

@@ -273,6 +273,21 @@ You should receive a JSON response. If not, confirm the file is correctly mounte
 
 See also: [SearXNG settings docs](https://docs.searxng.org/admin/settings/settings.html) · [discussion](https://github.com/searxng/searxng/discussions/1789)
 
+### Can't enable JSON? (HTML fallback)
+
+If you must use a public instance you don't control and it rejects `format=json` (the 403 above), set the opt-in flag instead of editing the server:
+
+```json
+"SEARXNG_HTML_FALLBACK": "true"
+```
+
+A search that gets a `403`/`404` or a non-JSON response is then retried automatically **without** `format=json` and parsed from the regular HTML results page.
+
+- **On success:** you get normal results (title, URL, snippet). They are marked `sourceFormat: "html"` in JSON mode, and text mode adds the line *"Note: Results parsed from SearXNG HTML fallback; metadata is limited."* Relevance scores and engine names are not available from HTML.
+- **On failure:** parsing is best-effort and varies by the instance's theme/version, so some results may be missed or sparse. If the HTML page itself also fails — still blocked, rate-limited (`429`), auth (`401`), or `5xx` — the **original error is surfaced unchanged**. The fallback only triggers on `403`/`404`/non-JSON, never on auth or network errors.
+
+Enabling JSON on an instance you control (above) remains the recommended setup — the fallback is a compatibility aid, not a replacement.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md)

package/dist/search.js

@@ -1,3 +1,4 @@
+import { parse } from "node-html-parser";
 import { getKnownCategories, getKnownEngines } from "./instance-info.js";
 import { createProxyAgent, createDefaultAgent, ProxyType } from "./proxy.js";
 import { logMessage } from "./logging.js";
@@ -32,6 +33,102 @@ function truncateResultContent(content, maxResultChars) {
     }
     return `${content.slice(0, maxResultChars)}…`;
 }
+function normalizeHtmlText(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+function isHtmlFallbackEnabled() {
+    return process.env.SEARXNG_HTML_FALLBACK === "true";
+}
+function shouldFallbackForStatus(status) {
+    return status === 403 || status === 404;
+}
+function buildHtmlFallbackUrl(jsonUrl) {
+    const htmlUrl = new URL(jsonUrl.toString());
+    htmlUrl.searchParams.delete("format");
+    return htmlUrl;
+}
+function parseHtmlSearchResults(html, query) {
+    const root = parse(html);
+    const articles = root.querySelectorAll("article.result");
+    const candidates = articles.length > 0 ? articles : root.querySelectorAll(".result");
+    const results = candidates
+        .map((entry) => {
+        const link = entry.querySelector("h3 > a") ?? entry.querySelector("h3 a") ?? entry.querySelector("a[href]");
+        if (!link) {
+            return undefined;
+        }
+        const href = link?.getAttribute("href")?.trim();
+        if (!href) {
+            return undefined;
+        }
+        try {
+            new URL(href);
+        }
+        catch {
+            return undefined;
+        }
+        const title = normalizeHtmlText(link.text);
+        const snippetNode = entry.querySelector("p.content") ?? entry.querySelector(".content");
+        const content = snippetNode ? normalizeHtmlText(snippetNode.text) : "";
+        return {
+            title,
+            url: href,
+            content,
+        };
+    })
+        .filter((result) => result !== undefined);
+    return {
+        query,
+        number_of_results: results.length,
+        results,
+        sourceFormat: "html",
+    };
+}
+async function fetchWithSearchTimeout(mcpServer, url, requestOptions, timeoutMs, query, searxngUrl) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        logMessage(mcpServer, "info", `Making request to: ${url.toString()}`);
+        return await fetch(url.toString(), {
+            ...requestOptions,
+            signal: controller.signal,
+        });
+    }
+    catch (error) {
+        logMessage(mcpServer, "error", `Network error during search request: ${error.message}`, { query, url: url.toString() });
+        const context = {
+            url: url.toString(),
+            searxngUrl,
+            proxyAgent: !!requestOptions.dispatcher,
+            username: process.env.AUTH_USERNAME,
+        };
+        throw createNetworkError(error, context);
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+async function fetchHtmlFallbackSearch(mcpServer, jsonUrl, requestOptions, timeoutMs, query, searxngUrl) {
+    const htmlUrl = buildHtmlFallbackUrl(jsonUrl);
+    logMessage(mcpServer, "info", `Retrying search with HTML fallback: ${htmlUrl.toString()}`);
+    const response = await fetchWithSearchTimeout(mcpServer, htmlUrl, requestOptions, timeoutMs, query, searxngUrl);
+    if (!response.ok) {
+        let responseBody;
+        try {
+            responseBody = await response.text();
+        }
+        catch {
+            responseBody = '[Could not read response body]';
+        }
+        const context = {
+            url: htmlUrl.toString(),
+            searxngUrl,
+        };
+        throw createServerError(response.status, response.statusText, responseBody, context);
+    }
+    const html = await response.text();
+    return parseHtmlSearchResults(html, query);
+}
 function hasItems(items) {
     return Array.isArray(items) && items.length > 0;
 }
@@ -278,57 +375,49 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
     }
     // Fetch with AbortController timeout and enhanced error handling
     const SEARCH_TIMEOUT_MS = parseInt(process.env.SEARXNG_TIMEOUT_MS ?? "10000", 10);
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), SEARCH_TIMEOUT_MS);
     let response;
-    try {
-        logMessage(mcpServer, "info", `Making request to: ${url.toString()}`);
-        response = await fetch(url.toString(), {
-            ...requestOptions,
-            signal: controller.signal,
-        });
-    }
-    catch (error) {
-        clearTimeout(timeoutId);
-        logMessage(mcpServer, "error", `Network error during search request: ${error.message}`, { query, url: url.toString() });
-        const context = {
-            url: url.toString(),
-            searxngUrl,
-            proxyAgent: !!dispatcher,
-            username
-        };
-        throw createNetworkError(error, context);
-    }
-    clearTimeout(timeoutId);
+    response = await fetchWithSearchTimeout(mcpServer, url, requestOptions, SEARCH_TIMEOUT_MS, query, searxngUrl);
+    let data;
     if (!response.ok) {
-        let responseBody;
-        try {
-            responseBody = await response.text();
+        if (isHtmlFallbackEnabled() && shouldFallbackForStatus(response.status)) {
+            data = await fetchHtmlFallbackSearch(mcpServer, url, requestOptions, SEARCH_TIMEOUT_MS, query, searxngUrl);
         }
-        catch {
-            responseBody = '[Could not read response body]';
+        else {
+            let responseBody;
+            try {
+                responseBody = await response.text();
+            }
+            catch {
+                responseBody = '[Could not read response body]';
+            }
+            const context = {
+                url: url.toString(),
+                searxngUrl
+            };
+            throw createServerError(response.status, response.statusText, responseBody, context);
         }
-        const context = {
-            url: url.toString(),
-            searxngUrl
-        };
-        throw createServerError(response.status, response.statusText, responseBody, context);
-    }
-    // Parse JSON response
-    let data;
-    try {
-        data = (await response.json());
     }
-    catch (error) {
-        let responseText;
+    else {
+        // Parse JSON response
         try {
-            responseText = await response.text();
+            data = (await response.json());
         }
-        catch {
-            responseText = '[Could not read response text]';
+        catch (error) {
+            if (isHtmlFallbackEnabled()) {
+                data = await fetchHtmlFallbackSearch(mcpServer, url, requestOptions, SEARCH_TIMEOUT_MS, query, searxngUrl);
+            }
+            else {
+                let responseText;
+                try {
+                    responseText = await response.text();
+                }
+                catch {
+                    responseText = '[Could not read response text]';
+                }
+                const context = { url: url.toString() };
+                throw createJSONError(responseText, context);
+            }
         }
-        const context = { url: url.toString() };
-        throw createJSONError(responseText, context);
     }
     if (!data.results) {
         const context = { url: url.toString(), query };
@@ -349,6 +438,7 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
     const metadata = formatSearchMetadata(data);
     const leadingSections = [
         filters.validationNote ?? null,
+        data.sourceFormat === "html" ? "Note: Results parsed from SearXNG HTML fallback; metadata is limited." : null,
         metadata || null,
     ].filter(Boolean).join("\n\n");
     if (slicedResults.length === 0) {
@@ -365,8 +455,15 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
     logMessage(mcpServer, "info", `Search completed: "${query}" (${searchParams}) - ${slicedResults.length} results in ${duration}ms`);
     const formattedResults = slicedResults
         .map((r) => {
-        const score = r.score || 0;
-        return `Title: ${r.title || ""}\nDescription: ${truncateResultContent(r.content || "", maxResultChars)}\nURL: ${r.url || ""}\nRelevance Score: ${score.toFixed(3)}`;
+        const lines = [
+            `Title: ${r.title || ""}`,
+            `Description: ${truncateResultContent(r.content || "", maxResultChars)}`,
+            `URL: ${r.url || ""}`,
+        ];
+        if (r.score !== undefined) {
+            lines.push(`Relevance Score: ${r.score.toFixed(3)}`);
+        }
+        return lines.join("\n");
     })
         .join("\n\n");
     return leadingSections ? `${leadingSections}\n\n---\n\n${formattedResults}` : formattedResults;

package/dist/types.d.ts

@@ -3,7 +3,7 @@ export interface SearXNGWebResult {
     title: string;
     content: string;
     url: string;
-    score: number;
+    score?: number;
     engine?: string;
     engines?: string[];
     category?: string;
@@ -23,6 +23,7 @@ export interface SearXNGWeb {
     query: string;
     number_of_results: number;
     results: SearXNGWebResult[];
+    sourceFormat?: "json" | "html";
     suggestions?: string[];
     corrections?: string[];
     answers?: string[];

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const packageVersion = "1.6.0";
+export declare const packageVersion = "1.7.0";

package/dist/version.js

@@ -1 +1 @@
-export const packageVersion = "1.6.0";
+export const packageVersion = "1.7.0";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcp-searxng",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "mcpName": "io.github.ihor-sokoliuk/mcp-searxng",
     "description": "MCP server for SearXNG integration",
     "license": "MIT",
@@ -55,16 +55,17 @@
         "express": "^5.2.1",
         "express-rate-limit": "^8.5.2",
         "node-html-markdown": "^2.0.0",
-        "undici": "7.27.2"
+        "node-html-parser": "^6.1.13",
+        "undici": "7.28.0"
     },
     "devDependencies": {
-        "@types/node": "22.19.20",
+        "@types/node": "22.19.21",
         "@types/supertest": "^7.2.0",
-        "@typescript-eslint/eslint-plugin": "8.61.0",
-        "@typescript-eslint/parser": "8.61.0",
+        "@typescript-eslint/eslint-plugin": "8.61.1",
+        "@typescript-eslint/parser": "8.61.1",
         "c8": "^11.0.0",
         "cross-env": "^10.1.0",
-        "eslint": "10.4.1",
+        "eslint": "10.5.0",
         "eslint-plugin-security": "^4.0.0",
         "fast-check": "^4.8.0",
         "shx": "^0.4.0",