mcp-searxng 1.3.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/index.js +65 -5
- package/dist/instance-info.d.ts +3 -0
- package/dist/instance-info.js +113 -0
- package/dist/resources.js +46 -3
- package/dist/search.d.ts +1 -1
- package/dist/search.js +119 -20
- package/dist/suggestions.d.ts +2 -0
- package/dist/suggestions.js +34 -0
- package/dist/types.d.ts +19 -0
- package/dist/types.js +157 -0
- package/dist/url-reader.d.ts +3 -0
- package/dist/url-reader.js +54 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -40,6 +40,10 @@ Replace `YOUR_SEARXNG_INSTANCE_URL` with the URL of your SearXNG instance (e.g.
|
|
|
40
40
|
## Features
|
|
41
41
|
|
|
42
42
|
- **Web Search**: General queries, news, articles, with pagination.
|
|
43
|
+
- **Structured Search Output**: Choose formatted text or raw SearXNG-shaped JSON with `response_format`.
|
|
44
|
+
- **Direct Answers & Metadata**: Text results surface SearXNG answers, corrections, suggestions, and infoboxes before result lists.
|
|
45
|
+
- **Search Suggestions**: Query autocomplete via SearXNG's `/autocompleter` endpoint.
|
|
46
|
+
- **Instance Capability Discovery**: Inspect configured categories, engines, defaults, locales, and plugins from `/config`.
|
|
43
47
|
- **URL Content Reading**: Advanced content extraction with pagination, section filtering, and heading extraction.
|
|
44
48
|
- **Intelligent Caching**: URL content is cached with TTL (Time-To-Live) to improve performance and reduce redundant requests.
|
|
45
49
|
- **Pagination**: Control which page of results to retrieve.
|
|
@@ -86,6 +90,23 @@ AI Assistant (e.g. Claude)
|
|
|
86
90
|
- `language` (string, optional): Language code for results (e.g., "en", "fr", "de") or "all" (default: "all")
|
|
87
91
|
- `safesearch` (number, optional): Safe search filter level (0: None, 1: Moderate, 2: Strict) (default: instance setting)
|
|
88
92
|
- `min_score` (number, optional): Minimum relevance score from 0.0 to 1.0. Results below this score are filtered out.
|
|
93
|
+
- `num_results` (number, optional): Maximum number of results to return, from 1 to 20. `SEARXNG_MAX_RESULTS` applies as an operator ceiling.
|
|
94
|
+
- `categories` (string, optional): Comma-separated SearXNG categories (e.g. `"news"`, `"it,science"`). Supported values: `general`, `news`, `images`, `videos`, `it`, `science`, `files`, `social media`. Default: SearXNG instance default (usually `general`).
|
|
95
|
+
- `response_format` (string, optional): Response format, either `"text"` for formatted agent-readable output or `"json"` for raw SearXNG JSON with filtered/sliced `results`. (default: `"text"`)
|
|
96
|
+
|
|
97
|
+
- **searxng_search_suggestions**
|
|
98
|
+
- Get autocomplete suggestions for refining search queries
|
|
99
|
+
- Inputs:
|
|
100
|
+
- `query` (string): Partial or complete query to autocomplete.
|
|
101
|
+
- `language` (string, optional): Language code for suggestions (e.g., "en", "fr", "de") or "all" (default: "all")
|
|
102
|
+
|
|
103
|
+
- **searxng_instance_info**
|
|
104
|
+
- Discover categories, engines, defaults, locales, and plugins exposed by the configured SearXNG instance
|
|
105
|
+
- Inputs:
|
|
106
|
+
- `includeEngines` (boolean, optional): Include enabled engine names in the response. (default: false)
|
|
107
|
+
- `includeDisabled` (boolean, optional): Include disabled engine names when `includeEngines` is true. (default: false)
|
|
108
|
+
- `category` (string, optional): Filter categories and engines to a single category name.
|
|
109
|
+
- `refresh` (boolean, optional): Bypass the process cache and fetch fresh `/config` data. (default: false)
|
|
89
110
|
|
|
90
111
|
- **web_url_read**
|
|
91
112
|
- Read and convert the content from a URL to markdown with advanced content extraction options
|
package/dist/index.js
CHANGED
|
@@ -2,9 +2,11 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
3
|
import { CallToolRequestSchema, ListToolsRequestSchema, SetLevelRequestSchema, ListResourcesRequestSchema, ListResourceTemplatesRequestSchema, ReadResourceRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
4
4
|
// Import modularized functionality
|
|
5
|
-
import { WEB_SEARCH_TOOL, READ_URL_TOOL, isSearXNGWebSearchArgs } from "./types.js";
|
|
5
|
+
import { WEB_SEARCH_TOOL, SUGGESTIONS_TOOL, INSTANCE_INFO_TOOL, READ_URL_TOOL, LITE_WEB_SEARCH_TOOL, LITE_SUGGESTIONS_TOOL, LITE_INSTANCE_INFO_TOOL, LITE_READ_URL_TOOL, isSearXNGWebSearchArgs, isSearXNGSearchSuggestionsArgs, isSearXNGInstanceInfoArgs, } from "./types.js";
|
|
6
6
|
import { logMessage, setLogLevel, getCurrentLogLevel } from "./logging.js";
|
|
7
7
|
import { performWebSearch } from "./search.js";
|
|
8
|
+
import { performSearchSuggestions } from "./suggestions.js";
|
|
9
|
+
import { fetchInstanceInfo } from "./instance-info.js";
|
|
8
10
|
import { fetchAndConvertToMarkdown } from "./url-reader.js";
|
|
9
11
|
import { createConfigResource, createHelpResource } from "./resources.js";
|
|
10
12
|
import { createHttpServer, resolveBindHost } from "./http-server.js";
|
|
@@ -41,6 +43,30 @@ export function isWebUrlReadArgs(args) {
|
|
|
41
43
|
}
|
|
42
44
|
return true;
|
|
43
45
|
}
|
|
46
|
+
function getFetchTimeoutMs(mcpServer) {
|
|
47
|
+
const rawValue = process.env.FETCH_TIMEOUT_MS;
|
|
48
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
49
|
+
return 10000;
|
|
50
|
+
}
|
|
51
|
+
const parsed = parseInt(rawValue, 10);
|
|
52
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
53
|
+
logMessage(mcpServer, "warning", `Ignoring invalid FETCH_TIMEOUT_MS="${rawValue}". Expected a positive integer. Using default 10000.`);
|
|
54
|
+
return 10000;
|
|
55
|
+
}
|
|
56
|
+
return parsed;
|
|
57
|
+
}
|
|
58
|
+
function getDefaultUrlReadMaxChars(mcpServer) {
|
|
59
|
+
const rawValue = process.env.URL_READ_MAX_CHARS;
|
|
60
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
const parsed = parseInt(rawValue, 10);
|
|
64
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
65
|
+
logMessage(mcpServer, "warning", `Ignoring invalid URL_READ_MAX_CHARS="${rawValue}". Expected a positive integer.`);
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
return parsed;
|
|
69
|
+
}
|
|
44
70
|
/**
|
|
45
71
|
* Creates and configures a new McpServer with all handlers registered.
|
|
46
72
|
* Called once per HTTP session, or once for STDIO mode.
|
|
@@ -57,11 +83,16 @@ export function createMcpServer() {
|
|
|
57
83
|
},
|
|
58
84
|
});
|
|
59
85
|
const server = mcpServer.server;
|
|
86
|
+
const useLiteTools = process.env.SEARXNG_LITE_TOOLS === "true";
|
|
87
|
+
const searchTool = useLiteTools ? LITE_WEB_SEARCH_TOOL : WEB_SEARCH_TOOL;
|
|
88
|
+
const suggestionsTool = useLiteTools ? LITE_SUGGESTIONS_TOOL : SUGGESTIONS_TOOL;
|
|
89
|
+
const instanceInfoTool = useLiteTools ? LITE_INSTANCE_INFO_TOOL : INSTANCE_INFO_TOOL;
|
|
90
|
+
const readUrlTool = useLiteTools ? LITE_READ_URL_TOOL : READ_URL_TOOL;
|
|
60
91
|
// List tools handler
|
|
61
92
|
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
62
93
|
logMessage(mcpServer, "debug", "Handling list_tools request");
|
|
63
94
|
return {
|
|
64
|
-
tools: [
|
|
95
|
+
tools: [searchTool, suggestionsTool, instanceInfoTool, readUrlTool],
|
|
65
96
|
};
|
|
66
97
|
});
|
|
67
98
|
// Call tool handler
|
|
@@ -73,7 +104,35 @@ export function createMcpServer() {
|
|
|
73
104
|
if (!isSearXNGWebSearchArgs(args)) {
|
|
74
105
|
throw new Error("Invalid arguments for web search");
|
|
75
106
|
}
|
|
76
|
-
const result = await performWebSearch(mcpServer, args.query, args.pageno, args.time_range, args.language, args.safesearch, args.min_score);
|
|
107
|
+
const result = await performWebSearch(mcpServer, args.query, args.pageno, args.time_range, args.language, args.safesearch, args.min_score, args.num_results, args.categories, args.response_format);
|
|
108
|
+
return {
|
|
109
|
+
content: [
|
|
110
|
+
{
|
|
111
|
+
type: "text",
|
|
112
|
+
text: result,
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
else if (name === "searxng_search_suggestions") {
|
|
118
|
+
if (!isSearXNGSearchSuggestionsArgs(args)) {
|
|
119
|
+
throw new Error("Invalid arguments for search suggestions");
|
|
120
|
+
}
|
|
121
|
+
const suggestions = await performSearchSuggestions(mcpServer, args.query, args.language);
|
|
122
|
+
return {
|
|
123
|
+
content: [
|
|
124
|
+
{
|
|
125
|
+
type: "text",
|
|
126
|
+
text: JSON.stringify({ query: args.query, suggestions }, null, 2),
|
|
127
|
+
},
|
|
128
|
+
],
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
else if (name === "searxng_instance_info") {
|
|
132
|
+
if (!isSearXNGInstanceInfoArgs(args)) {
|
|
133
|
+
throw new Error("Invalid arguments for instance info");
|
|
134
|
+
}
|
|
135
|
+
const result = await fetchInstanceInfo(mcpServer, args.includeEngines, args.includeDisabled, args.category, args.refresh);
|
|
77
136
|
return {
|
|
78
137
|
content: [
|
|
79
138
|
{
|
|
@@ -87,14 +146,15 @@ export function createMcpServer() {
|
|
|
87
146
|
if (!isWebUrlReadArgs(args)) {
|
|
88
147
|
throw new Error("Invalid arguments for URL reading");
|
|
89
148
|
}
|
|
149
|
+
const defaultMaxLength = getDefaultUrlReadMaxChars(mcpServer);
|
|
90
150
|
const paginationOptions = {
|
|
91
151
|
startChar: args.startChar,
|
|
92
|
-
maxLength: args.maxLength,
|
|
152
|
+
maxLength: args.maxLength ?? defaultMaxLength,
|
|
93
153
|
section: args.section,
|
|
94
154
|
paragraphRange: args.paragraphRange,
|
|
95
155
|
readHeadings: args.readHeadings,
|
|
96
156
|
};
|
|
97
|
-
const result = await fetchAndConvertToMarkdown(mcpServer, args.url,
|
|
157
|
+
const result = await fetchAndConvertToMarkdown(mcpServer, args.url, getFetchTimeoutMs(mcpServer), paginationOptions);
|
|
98
158
|
return {
|
|
99
159
|
content: [
|
|
100
160
|
{
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
export declare function clearInstanceInfoCacheForTests(): void;
|
|
3
|
+
export declare function fetchInstanceInfo(mcpServer: McpServer, includeEngines?: boolean, includeDisabled?: boolean, category?: string, refresh?: boolean): Promise<string>;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { logMessage } from "./logging.js";
|
|
2
|
+
import { createDefaultAgent, createProxyAgent, ProxyType } from "./proxy.js";
|
|
3
|
+
let cachedConfig = null;
|
|
4
|
+
let cachedBaseUrl = null;
|
|
5
|
+
function unavailable(message, status) {
|
|
6
|
+
return JSON.stringify({
|
|
7
|
+
available: false,
|
|
8
|
+
message,
|
|
9
|
+
...(status !== undefined ? { status } : {}),
|
|
10
|
+
}, null, 2);
|
|
11
|
+
}
|
|
12
|
+
function namesFromCategories(config) {
|
|
13
|
+
if (!config.categories || typeof config.categories !== "object") {
|
|
14
|
+
return [];
|
|
15
|
+
}
|
|
16
|
+
return Object.keys(config.categories).sort();
|
|
17
|
+
}
|
|
18
|
+
function engineCategories(engine) {
|
|
19
|
+
if (Array.isArray(engine.categories)) {
|
|
20
|
+
return engine.categories;
|
|
21
|
+
}
|
|
22
|
+
if (typeof engine.category === "string") {
|
|
23
|
+
return [engine.category];
|
|
24
|
+
}
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
function collectEngines(config, includeDisabled, category) {
|
|
28
|
+
const enabled = new Set();
|
|
29
|
+
const disabled = new Set();
|
|
30
|
+
if (Array.isArray(config.engines)) {
|
|
31
|
+
for (const engine of config.engines) {
|
|
32
|
+
if (!engine || typeof engine.name !== "string") {
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
const categories = engineCategories(engine);
|
|
36
|
+
if (category && !categories.includes(category)) {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (engine.disabled) {
|
|
40
|
+
if (includeDisabled) {
|
|
41
|
+
disabled.add(engine.name);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
enabled.add(engine.name);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
enabled: [...enabled].sort(),
|
|
51
|
+
...(includeDisabled ? { disabled: [...disabled].sort() } : {}),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function formatInstanceInfo(config, includeEngines, includeDisabled, category) {
|
|
55
|
+
const categories = category
|
|
56
|
+
? namesFromCategories(config).filter((name) => name === category)
|
|
57
|
+
: namesFromCategories(config);
|
|
58
|
+
const payload = {
|
|
59
|
+
available: true,
|
|
60
|
+
categories,
|
|
61
|
+
defaults: {
|
|
62
|
+
safesearch: config.search?.safe_search ?? config.default_safe_search,
|
|
63
|
+
locale: config.default_locale,
|
|
64
|
+
language: config.default_language,
|
|
65
|
+
theme: config.default_theme,
|
|
66
|
+
},
|
|
67
|
+
locales: config.locales,
|
|
68
|
+
plugins: config.plugins ?? [],
|
|
69
|
+
};
|
|
70
|
+
if (includeEngines) {
|
|
71
|
+
payload.engines = collectEngines(config, includeDisabled, category);
|
|
72
|
+
}
|
|
73
|
+
return JSON.stringify(payload, null, 2);
|
|
74
|
+
}
|
|
75
|
+
export function clearInstanceInfoCacheForTests() {
|
|
76
|
+
cachedConfig = null;
|
|
77
|
+
cachedBaseUrl = null;
|
|
78
|
+
}
|
|
79
|
+
export async function fetchInstanceInfo(mcpServer, includeEngines = false, includeDisabled = false, category, refresh = false) {
|
|
80
|
+
const base = process.env.SEARXNG_URL;
|
|
81
|
+
if (!base) {
|
|
82
|
+
return unavailable("SEARXNG_URL is not configured; cannot fetch SearXNG /config.");
|
|
83
|
+
}
|
|
84
|
+
if (refresh) {
|
|
85
|
+
cachedConfig = null;
|
|
86
|
+
}
|
|
87
|
+
if (cachedConfig && cachedBaseUrl === base) {
|
|
88
|
+
return formatInstanceInfo(cachedConfig, includeEngines, includeDisabled, category);
|
|
89
|
+
}
|
|
90
|
+
const parsedBase = new URL(base.endsWith("/") ? base : `${base}/`);
|
|
91
|
+
const url = new URL("config", parsedBase);
|
|
92
|
+
try {
|
|
93
|
+
const requestOptions = {
|
|
94
|
+
signal: AbortSignal.timeout(5000),
|
|
95
|
+
};
|
|
96
|
+
const proxyAgent = createProxyAgent(url.toString(), ProxyType.SEARCH);
|
|
97
|
+
const dispatcher = proxyAgent ?? createDefaultAgent();
|
|
98
|
+
if (dispatcher) {
|
|
99
|
+
requestOptions.dispatcher = dispatcher;
|
|
100
|
+
}
|
|
101
|
+
const response = await fetch(url.toString(), requestOptions);
|
|
102
|
+
if (!response.ok) {
|
|
103
|
+
return unavailable(`SearXNG /config is unavailable: HTTP ${response.status} ${response.statusText}`, response.status);
|
|
104
|
+
}
|
|
105
|
+
cachedConfig = await response.json();
|
|
106
|
+
cachedBaseUrl = base;
|
|
107
|
+
return formatInstanceInfo(cachedConfig, includeEngines, includeDisabled, category);
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
logMessage(mcpServer, "warning", `SearXNG /config fetch failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
111
|
+
return unavailable("SearXNG /config is unavailable; instance capability discovery could not complete.");
|
|
112
|
+
}
|
|
113
|
+
}
|
package/dist/resources.js
CHANGED
|
@@ -21,7 +21,12 @@ export function createConfigResource() {
|
|
|
21
21
|
currentLogLevel: getCurrentLogLevel()
|
|
22
22
|
},
|
|
23
23
|
capabilities: {
|
|
24
|
-
tools: [
|
|
24
|
+
tools: [
|
|
25
|
+
"searxng_web_search",
|
|
26
|
+
"searxng_search_suggestions",
|
|
27
|
+
"searxng_instance_info",
|
|
28
|
+
"web_url_read"
|
|
29
|
+
],
|
|
25
30
|
logging: true,
|
|
26
31
|
resources: true,
|
|
27
32
|
transports: process.env.MCP_HTTP_PORT ? ["stdio", "http"] : ["stdio"]
|
|
@@ -33,7 +38,7 @@ export function createHelpResource() {
|
|
|
33
38
|
return `# SearXNG MCP Server Help
|
|
34
39
|
|
|
35
40
|
## Overview
|
|
36
|
-
This is a Model Context Protocol (MCP) server that provides web search
|
|
41
|
+
This is a Model Context Protocol (MCP) server that provides web search, autocomplete suggestions, instance capability discovery, and URL content reading through SearXNG.
|
|
37
42
|
|
|
38
43
|
## Available Tools
|
|
39
44
|
|
|
@@ -47,12 +52,38 @@ Performs web searches using the configured SearXNG instance.
|
|
|
47
52
|
- \`language\` (optional): Language code like "en", "fr", "de" (default: "all")
|
|
48
53
|
- \`safesearch\` (optional): Safe search level - 0 (none), 1 (moderate), 2 (strict)
|
|
49
54
|
- \`min_score\` (optional): Minimum relevance score from 0.0 to 1.0
|
|
55
|
+
- \`num_results\` (optional): Maximum result count from 1 to 20
|
|
56
|
+
- \`categories\` (optional): Comma-separated SearXNG categories such as "news" or "it,science"
|
|
57
|
+
- \`response_format\` (optional): "text" for formatted output or "json" for raw SearXNG-shaped JSON
|
|
50
58
|
|
|
51
|
-
|
|
59
|
+
Text output can include metadata sections for direct answers, spelling corrections, suggestions, and infoboxes before the result list. JSON output preserves the SearXNG response shape with filtered and sliced \`results\`.
|
|
60
|
+
|
|
61
|
+
### 2. searxng_search_suggestions
|
|
62
|
+
Returns autocomplete suggestions from the configured SearXNG instance.
|
|
63
|
+
|
|
64
|
+
**Parameters:**
|
|
65
|
+
- \`query\` (required): Partial or complete query to autocomplete
|
|
66
|
+
- \`language\` (optional): Language code like "en", "fr", "de" or "all" (default: "all")
|
|
67
|
+
|
|
68
|
+
### 3. searxng_instance_info
|
|
69
|
+
Discovers categories, engines, defaults, locales, and plugins exposed by the configured SearXNG instance.
|
|
70
|
+
|
|
71
|
+
**Parameters:**
|
|
72
|
+
- \`includeEngines\` (optional): Include enabled engine names
|
|
73
|
+
- \`includeDisabled\` (optional): Include disabled engine names when \`includeEngines\` is true
|
|
74
|
+
- \`category\` (optional): Filter categories and engines to one category
|
|
75
|
+
- \`refresh\` (optional): Bypass the process cache and fetch fresh \`/config\` data
|
|
76
|
+
|
|
77
|
+
### 4. web_url_read
|
|
52
78
|
Reads and converts web page content to Markdown format.
|
|
53
79
|
|
|
54
80
|
**Parameters:**
|
|
55
81
|
- \`url\` (required): The URL to fetch and convert
|
|
82
|
+
- \`startChar\` (optional): Starting character position
|
|
83
|
+
- \`maxLength\` (optional): Maximum number of characters to return
|
|
84
|
+
- \`section\` (optional): Extract content under a heading
|
|
85
|
+
- \`paragraphRange\` (optional): Return a paragraph range such as "1-5" or "10-"
|
|
86
|
+
- \`readHeadings\` (optional): Return only headings
|
|
56
87
|
|
|
57
88
|
## Configuration
|
|
58
89
|
|
|
@@ -98,6 +129,18 @@ Tool: web_url_read
|
|
|
98
129
|
Args: {"url": "https://example.com/article"}
|
|
99
130
|
\`\`\`
|
|
100
131
|
|
|
132
|
+
### Get query suggestions
|
|
133
|
+
\`\`\`
|
|
134
|
+
Tool: searxng_search_suggestions
|
|
135
|
+
Args: {"query": "typescr"}
|
|
136
|
+
\`\`\`
|
|
137
|
+
|
|
138
|
+
### Discover instance capabilities
|
|
139
|
+
\`\`\`
|
|
140
|
+
Tool: searxng_instance_info
|
|
141
|
+
Args: {"includeEngines": true}
|
|
142
|
+
\`\`\`
|
|
143
|
+
|
|
101
144
|
## Troubleshooting
|
|
102
145
|
|
|
103
146
|
1. **"SEARXNG_URL not set"**: Configure the SEARXNG_URL environment variable
|
package/dist/search.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
-
export declare function performWebSearch(mcpServer: McpServer, query: string, pageno?: number, time_range?: string, language?: string, safesearch?: number, min_score?: number): Promise<string>;
|
|
2
|
+
export declare function performWebSearch(mcpServer: McpServer, query: string, pageno?: number, time_range?: string, language?: string, safesearch?: number, min_score?: number, num_results?: number, categories?: string, response_format?: "text" | "json"): Promise<string>;
|
package/dist/search.js
CHANGED
|
@@ -1,14 +1,100 @@
|
|
|
1
1
|
import { createProxyAgent, createDefaultAgent, ProxyType } from "./proxy.js";
|
|
2
2
|
import { logMessage } from "./logging.js";
|
|
3
3
|
import { MCPSearXNGError, validateEnvironment, createNetworkError, createServerError, createJSONError, createDataError, createNoResultsMessage } from "./error-handler.js";
|
|
4
|
-
|
|
4
|
+
function getOperatorMaxResults(mcpServer) {
|
|
5
|
+
const rawValue = process.env.SEARXNG_MAX_RESULTS;
|
|
6
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
7
|
+
return undefined;
|
|
8
|
+
}
|
|
9
|
+
const parsed = parseInt(rawValue, 10);
|
|
10
|
+
if (Number.isNaN(parsed) || parsed <= 0 || parsed > 20) {
|
|
11
|
+
logMessage(mcpServer, "warning", `Ignoring invalid SEARXNG_MAX_RESULTS="${rawValue}". Expected an integer from 1 to 20.`);
|
|
12
|
+
return undefined;
|
|
13
|
+
}
|
|
14
|
+
return parsed;
|
|
15
|
+
}
|
|
16
|
+
function getMaxResultChars(mcpServer) {
|
|
17
|
+
const rawValue = process.env.SEARXNG_MAX_RESULT_CHARS;
|
|
18
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
const parsed = parseInt(rawValue, 10);
|
|
22
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
23
|
+
logMessage(mcpServer, "warning", `Ignoring invalid SEARXNG_MAX_RESULT_CHARS="${rawValue}". Expected a positive integer.`);
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
return parsed;
|
|
27
|
+
}
|
|
28
|
+
function truncateResultContent(content, maxResultChars) {
|
|
29
|
+
if (maxResultChars === undefined || content.length <= maxResultChars) {
|
|
30
|
+
return content;
|
|
31
|
+
}
|
|
32
|
+
return `${content.slice(0, maxResultChars)}…`;
|
|
33
|
+
}
|
|
34
|
+
function hasItems(items) {
|
|
35
|
+
return Array.isArray(items) && items.length > 0;
|
|
36
|
+
}
|
|
37
|
+
function formatSearchMetadata(data) {
|
|
38
|
+
const sections = [];
|
|
39
|
+
if (hasItems(data.answers)) {
|
|
40
|
+
sections.push(data.answers.map((answer) => `Direct answer: ${answer}`).join("\n"));
|
|
41
|
+
}
|
|
42
|
+
if (hasItems(data.corrections)) {
|
|
43
|
+
sections.push(data.corrections.map((correction) => `Spelling correction: did you mean "${correction}"?`).join("\n"));
|
|
44
|
+
}
|
|
45
|
+
if (hasItems(data.suggestions)) {
|
|
46
|
+
sections.push(`Suggestions: ${data.suggestions.join(", ")}`);
|
|
47
|
+
}
|
|
48
|
+
if (hasItems(data.infoboxes)) {
|
|
49
|
+
const infoboxText = data.infoboxes
|
|
50
|
+
.map((infobox) => {
|
|
51
|
+
const lines = [`Infobox: ${infobox.infobox}`];
|
|
52
|
+
if (infobox.content) {
|
|
53
|
+
lines.push(infobox.content);
|
|
54
|
+
}
|
|
55
|
+
if (hasItems(infobox.urls)) {
|
|
56
|
+
lines.push(...infobox.urls.map((entry) => `${entry.title}: ${entry.url}`));
|
|
57
|
+
}
|
|
58
|
+
return lines.join("\n");
|
|
59
|
+
})
|
|
60
|
+
.join("\n\n");
|
|
61
|
+
sections.push(infoboxText);
|
|
62
|
+
}
|
|
63
|
+
return sections.join("\n\n");
|
|
64
|
+
}
|
|
65
|
+
function getDefaultLanguage() {
|
|
66
|
+
return process.env.SEARXNG_DEFAULT_LANGUAGE ?? "all";
|
|
67
|
+
}
|
|
68
|
+
function getDefaultSafesearch(mcpServer) {
|
|
69
|
+
const rawValue = process.env.SEARXNG_DEFAULT_SAFESEARCH;
|
|
70
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
const parsed = parseInt(rawValue, 10);
|
|
74
|
+
if (Number.isNaN(parsed) || ![0, 1, 2].includes(parsed)) {
|
|
75
|
+
logMessage(mcpServer, "warning", `Ignoring invalid SEARXNG_DEFAULT_SAFESEARCH="${rawValue}". Expected 0, 1, or 2.`);
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
return parsed;
|
|
79
|
+
}
|
|
80
|
+
export async function performWebSearch(mcpServer, query, pageno = 1, time_range, language, safesearch, min_score, num_results, categories, response_format = "text") {
|
|
5
81
|
const startTime = Date.now();
|
|
82
|
+
const operatorMax = getOperatorMaxResults(mcpServer);
|
|
83
|
+
const effectiveMax = operatorMax !== undefined
|
|
84
|
+
? (num_results !== undefined ? Math.min(num_results, operatorMax) : operatorMax)
|
|
85
|
+
: num_results;
|
|
86
|
+
const maxResultChars = getMaxResultChars(mcpServer);
|
|
87
|
+
const effectiveLanguage = language ?? getDefaultLanguage();
|
|
88
|
+
const effectiveSafesearch = safesearch !== undefined ? safesearch : getDefaultSafesearch(mcpServer);
|
|
6
89
|
// Build detailed log message with all parameters
|
|
7
90
|
const searchParams = [
|
|
8
91
|
`page ${pageno}`,
|
|
9
|
-
`lang: ${
|
|
92
|
+
`lang: ${effectiveLanguage}`,
|
|
10
93
|
time_range ? `time: ${time_range}` : null,
|
|
11
|
-
|
|
94
|
+
effectiveSafesearch !== undefined ? `safesearch: ${effectiveSafesearch}` : null,
|
|
95
|
+
min_score !== undefined ? `min_score: ${min_score}` : null,
|
|
96
|
+
effectiveMax !== undefined ? `num_results: ${effectiveMax}` : null,
|
|
97
|
+
categories ? `categories: ${categories}` : null,
|
|
12
98
|
].filter(Boolean).join(", ");
|
|
13
99
|
logMessage(mcpServer, "info", `Starting web search: "${query}" (${searchParams})`);
|
|
14
100
|
const validationError = validateEnvironment();
|
|
@@ -26,11 +112,14 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
|
|
|
26
112
|
["day", "week", "month", "year"].includes(time_range)) {
|
|
27
113
|
url.searchParams.set("time_range", time_range);
|
|
28
114
|
}
|
|
29
|
-
if (
|
|
30
|
-
url.searchParams.set("language",
|
|
115
|
+
if (effectiveLanguage && effectiveLanguage !== "all") {
|
|
116
|
+
url.searchParams.set("language", effectiveLanguage);
|
|
31
117
|
}
|
|
32
|
-
if (
|
|
33
|
-
url.searchParams.set("safesearch",
|
|
118
|
+
if (effectiveSafesearch !== undefined && [0, 1, 2].includes(effectiveSafesearch)) {
|
|
119
|
+
url.searchParams.set("safesearch", effectiveSafesearch.toString());
|
|
120
|
+
}
|
|
121
|
+
if (categories) {
|
|
122
|
+
url.searchParams.set("categories", categories);
|
|
34
123
|
}
|
|
35
124
|
// Prepare request options with headers
|
|
36
125
|
const requestOptions = {
|
|
@@ -119,21 +208,31 @@ export async function performWebSearch(mcpServer, query, pageno = 1, time_range,
|
|
|
119
208
|
throw createDataError(data, context);
|
|
120
209
|
}
|
|
121
210
|
const results = data.results
|
|
122
|
-
.
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
211
|
+
.filter((result) => min_score === undefined || (result.score || 0) >= min_score);
|
|
212
|
+
const slicedResults = effectiveMax !== undefined
|
|
213
|
+
? results.slice(0, effectiveMax)
|
|
214
|
+
: results;
|
|
215
|
+
if (response_format === "json") {
|
|
216
|
+
return JSON.stringify({ ...data, results: slicedResults }, null, 2);
|
|
217
|
+
}
|
|
218
|
+
const metadata = formatSearchMetadata(data);
|
|
219
|
+
if (slicedResults.length === 0) {
|
|
220
|
+
const appliedFilters = [
|
|
221
|
+
min_score === undefined ? null : `min_score=${min_score}`,
|
|
222
|
+
effectiveMax === undefined ? null : `num_results=${effectiveMax}`,
|
|
223
|
+
].filter(Boolean).join(" ");
|
|
224
|
+
const filterNote = appliedFilters ? ` after applying ${appliedFilters}` : "";
|
|
131
225
|
logMessage(mcpServer, "info", `No results found for query: "${query}"${filterNote}`);
|
|
132
|
-
|
|
226
|
+
const noResultsMessage = createNoResultsMessage(query);
|
|
227
|
+
return metadata ? `${metadata}\n\n---\n\n${noResultsMessage}` : noResultsMessage;
|
|
133
228
|
}
|
|
134
229
|
const duration = Date.now() - startTime;
|
|
135
|
-
logMessage(mcpServer, "info", `Search completed: "${query}" (${searchParams}) - ${
|
|
136
|
-
|
|
137
|
-
.map((r) =>
|
|
230
|
+
logMessage(mcpServer, "info", `Search completed: "${query}" (${searchParams}) - ${slicedResults.length} results in ${duration}ms`);
|
|
231
|
+
const formattedResults = slicedResults
|
|
232
|
+
.map((r) => {
|
|
233
|
+
const score = r.score || 0;
|
|
234
|
+
return `Title: ${r.title || ""}\nDescription: ${truncateResultContent(r.content || "", maxResultChars)}\nURL: ${r.url || ""}\nRelevance Score: ${score.toFixed(3)}`;
|
|
235
|
+
})
|
|
138
236
|
.join("\n\n");
|
|
237
|
+
return metadata ? `${metadata}\n\n---\n\n${formattedResults}` : formattedResults;
|
|
139
238
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { logMessage } from "./logging.js";
|
|
2
|
+
import { createDefaultAgent, createProxyAgent, ProxyType } from "./proxy.js";
|
|
3
|
+
export async function performSearchSuggestions(mcpServer, query, language = "all") {
|
|
4
|
+
const base = process.env.SEARXNG_URL;
|
|
5
|
+
if (!base) {
|
|
6
|
+
return [];
|
|
7
|
+
}
|
|
8
|
+
const parsedBase = new URL(base.endsWith("/") ? base : `${base}/`);
|
|
9
|
+
const url = new URL("autocompleter", parsedBase);
|
|
10
|
+
url.searchParams.set("q", query);
|
|
11
|
+
if (language !== "all") {
|
|
12
|
+
url.searchParams.set("lang", language);
|
|
13
|
+
}
|
|
14
|
+
try {
|
|
15
|
+
const requestOptions = {
|
|
16
|
+
signal: AbortSignal.timeout(5000),
|
|
17
|
+
};
|
|
18
|
+
const proxyAgent = createProxyAgent(url.toString(), ProxyType.SEARCH);
|
|
19
|
+
const dispatcher = proxyAgent ?? createDefaultAgent();
|
|
20
|
+
if (dispatcher) {
|
|
21
|
+
requestOptions.dispatcher = dispatcher;
|
|
22
|
+
}
|
|
23
|
+
const response = await fetch(url.toString(), requestOptions);
|
|
24
|
+
if (!response.ok) {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
const data = await response.json();
|
|
28
|
+
return Array.isArray(data[1]) ? data[1] : [];
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
logMessage(mcpServer, "debug", "Autocomplete request failed; returning empty suggestions");
|
|
32
|
+
return [];
|
|
33
|
+
}
|
|
34
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -36,6 +36,25 @@ export declare function isSearXNGWebSearchArgs(args: unknown): args is {
|
|
|
36
36
|
language?: string;
|
|
37
37
|
safesearch?: number;
|
|
38
38
|
min_score?: number;
|
|
39
|
+
num_results?: number;
|
|
40
|
+
categories?: string;
|
|
41
|
+
response_format?: "text" | "json";
|
|
42
|
+
};
|
|
43
|
+
export declare function isSearXNGSearchSuggestionsArgs(args: unknown): args is {
|
|
44
|
+
query: string;
|
|
45
|
+
language?: string;
|
|
46
|
+
};
|
|
47
|
+
export declare function isSearXNGInstanceInfoArgs(args: unknown): args is {
|
|
48
|
+
includeEngines?: boolean;
|
|
49
|
+
includeDisabled?: boolean;
|
|
50
|
+
category?: string;
|
|
51
|
+
refresh?: boolean;
|
|
39
52
|
};
|
|
40
53
|
export declare const WEB_SEARCH_TOOL: Tool;
|
|
54
|
+
export declare const SUGGESTIONS_TOOL: Tool;
|
|
55
|
+
export declare const INSTANCE_INFO_TOOL: Tool;
|
|
56
|
+
export declare const LITE_WEB_SEARCH_TOOL: Tool;
|
|
57
|
+
export declare const LITE_SUGGESTIONS_TOOL: Tool;
|
|
58
|
+
export declare const LITE_INSTANCE_INFO_TOOL: Tool;
|
|
59
|
+
export declare const LITE_READ_URL_TOOL: Tool;
|
|
41
60
|
export declare const READ_URL_TOOL: Tool;
|
package/dist/types.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const VALID_TIME_RANGES = ["day", "week", "month", "year"];
|
|
2
2
|
const VALID_SAFESEARCH_VALUES = [0, 1, 2];
|
|
3
|
+
const VALID_RESPONSE_FORMATS = ["text", "json"];
|
|
3
4
|
export function isSearXNGWebSearchArgs(args) {
|
|
4
5
|
if (typeof args !== "object" ||
|
|
5
6
|
args === null ||
|
|
@@ -29,6 +30,53 @@ export function isSearXNGWebSearchArgs(args) {
|
|
|
29
30
|
searchArgs.min_score > 1)) {
|
|
30
31
|
return false;
|
|
31
32
|
}
|
|
33
|
+
if (searchArgs.num_results !== undefined &&
|
|
34
|
+
(typeof searchArgs.num_results !== "number" ||
|
|
35
|
+
Number.isNaN(searchArgs.num_results) ||
|
|
36
|
+
!Number.isInteger(searchArgs.num_results) ||
|
|
37
|
+
searchArgs.num_results < 1 ||
|
|
38
|
+
searchArgs.num_results > 20)) {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
if (searchArgs.categories !== undefined && typeof searchArgs.categories !== "string") {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
if (searchArgs.response_format !== undefined &&
|
|
45
|
+
(typeof searchArgs.response_format !== "string" || !VALID_RESPONSE_FORMATS.includes(searchArgs.response_format))) {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
export function isSearXNGSearchSuggestionsArgs(args) {
|
|
51
|
+
if (typeof args !== "object" ||
|
|
52
|
+
args === null ||
|
|
53
|
+
!("query" in args) ||
|
|
54
|
+
typeof args.query !== "string") {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
const suggestionArgs = args;
|
|
58
|
+
if (suggestionArgs.language !== undefined && typeof suggestionArgs.language !== "string") {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
export function isSearXNGInstanceInfoArgs(args) {
|
|
64
|
+
if (typeof args !== "object" || args === null) {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
const infoArgs = args;
|
|
68
|
+
if (infoArgs.includeEngines !== undefined && typeof infoArgs.includeEngines !== "boolean") {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
if (infoArgs.includeDisabled !== undefined && typeof infoArgs.includeDisabled !== "boolean") {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
if (infoArgs.category !== undefined && typeof infoArgs.category !== "string") {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
if (infoArgs.refresh !== undefined && typeof infoArgs.refresh !== "boolean") {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
32
80
|
return true;
|
|
33
81
|
}
|
|
34
82
|
export const WEB_SEARCH_TOOL = {
|
|
@@ -76,10 +124,119 @@ export const WEB_SEARCH_TOOL = {
|
|
|
76
124
|
minimum: 0,
|
|
77
125
|
maximum: 1,
|
|
78
126
|
},
|
|
127
|
+
num_results: {
|
|
128
|
+
type: "number",
|
|
129
|
+
description: "Maximum number of results to return (1-20). Operator cap SEARXNG_MAX_RESULTS applies as a ceiling.",
|
|
130
|
+
minimum: 1,
|
|
131
|
+
maximum: 20,
|
|
132
|
+
},
|
|
133
|
+
categories: {
|
|
134
|
+
type: "string",
|
|
135
|
+
description: "Comma-separated SearXNG categories. Supported: general, news, images, videos, it, science, files, social media. Default: general (SearXNG instance default).",
|
|
136
|
+
},
|
|
137
|
+
response_format: {
|
|
138
|
+
type: "string",
|
|
139
|
+
description: "Response format: formatted text for agents or raw JSON for programmatic clients. Default: text.",
|
|
140
|
+
enum: ["text", "json"],
|
|
141
|
+
default: "text",
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
required: ["query"],
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
export const SUGGESTIONS_TOOL = {
|
|
148
|
+
name: "searxng_search_suggestions",
|
|
149
|
+
description: "Returns autocomplete suggestions from the configured SearXNG instance. " +
|
|
150
|
+
"Use this to refine vague or partial queries before searching.",
|
|
151
|
+
annotations: {
|
|
152
|
+
readOnlyHint: true,
|
|
153
|
+
openWorldHint: true,
|
|
154
|
+
},
|
|
155
|
+
inputSchema: {
|
|
156
|
+
type: "object",
|
|
157
|
+
properties: {
|
|
158
|
+
query: {
|
|
159
|
+
type: "string",
|
|
160
|
+
description: "Partial or complete query to autocomplete.",
|
|
161
|
+
},
|
|
162
|
+
language: {
|
|
163
|
+
type: "string",
|
|
164
|
+
description: "Language code for suggestions (e.g., 'en', 'fr', 'de') or 'all'. Default: all.",
|
|
165
|
+
default: "all",
|
|
166
|
+
},
|
|
167
|
+
},
|
|
168
|
+
required: ["query"],
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
export const INSTANCE_INFO_TOOL = {
|
|
172
|
+
name: "searxng_instance_info",
|
|
173
|
+
description: "Discovers capabilities from the configured SearXNG instance via /config, including categories, engines, defaults, locales, and plugins.",
|
|
174
|
+
annotations: {
|
|
175
|
+
readOnlyHint: true,
|
|
176
|
+
openWorldHint: true,
|
|
177
|
+
},
|
|
178
|
+
inputSchema: {
|
|
179
|
+
type: "object",
|
|
180
|
+
properties: {
|
|
181
|
+
includeEngines: {
|
|
182
|
+
type: "boolean",
|
|
183
|
+
description: "Include enabled engine names in the response.",
|
|
184
|
+
default: false,
|
|
185
|
+
},
|
|
186
|
+
includeDisabled: {
|
|
187
|
+
type: "boolean",
|
|
188
|
+
description: "Include disabled engine names when includeEngines is true.",
|
|
189
|
+
default: false,
|
|
190
|
+
},
|
|
191
|
+
category: {
|
|
192
|
+
type: "string",
|
|
193
|
+
description: "Filter categories and engines to a single category name.",
|
|
194
|
+
},
|
|
195
|
+
refresh: {
|
|
196
|
+
type: "boolean",
|
|
197
|
+
description: "Bypass the process cache and fetch fresh /config data.",
|
|
198
|
+
default: false,
|
|
199
|
+
},
|
|
79
200
|
},
|
|
201
|
+
required: [],
|
|
202
|
+
},
|
|
203
|
+
};
|
|
204
|
+
export const LITE_WEB_SEARCH_TOOL = {
|
|
205
|
+
name: "searxng_web_search",
|
|
206
|
+
description: "Web search. Returns titles, URLs, snippets.",
|
|
207
|
+
inputSchema: {
|
|
208
|
+
type: "object",
|
|
209
|
+
properties: { query: { type: "string", description: "Search query." } },
|
|
80
210
|
required: ["query"],
|
|
81
211
|
},
|
|
82
212
|
};
|
|
213
|
+
export const LITE_SUGGESTIONS_TOOL = {
|
|
214
|
+
name: "searxng_search_suggestions",
|
|
215
|
+
description: "Autocomplete search query suggestions.",
|
|
216
|
+
inputSchema: {
|
|
217
|
+
type: "object",
|
|
218
|
+
properties: { query: { type: "string", description: "Query prefix." } },
|
|
219
|
+
required: ["query"],
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
export const LITE_INSTANCE_INFO_TOOL = {
|
|
223
|
+
name: "searxng_instance_info",
|
|
224
|
+
description: "Discover SearXNG instance capabilities.",
|
|
225
|
+
inputSchema: {
|
|
226
|
+
type: "object",
|
|
227
|
+
properties: {},
|
|
228
|
+
required: [],
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
export const LITE_READ_URL_TOOL = {
|
|
232
|
+
name: "web_url_read",
|
|
233
|
+
description: "Fetch URL. Returns page text as markdown.",
|
|
234
|
+
inputSchema: {
|
|
235
|
+
type: "object",
|
|
236
|
+
properties: { url: { type: "string", description: "URL to fetch." } },
|
|
237
|
+
required: ["url"],
|
|
238
|
+
},
|
|
239
|
+
};
|
|
83
240
|
export const READ_URL_TOOL = {
|
|
84
241
|
name: "web_url_read",
|
|
85
242
|
description: "Fetches a URL and returns its text content converted to markdown. " +
|
package/dist/url-reader.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { type Dispatcher } from "undici";
|
|
2
3
|
interface PaginationOptions {
|
|
3
4
|
startChar?: number;
|
|
4
5
|
maxLength?: number;
|
|
@@ -6,5 +7,7 @@ interface PaginationOptions {
|
|
|
6
7
|
paragraphRange?: string;
|
|
7
8
|
readHeadings?: boolean;
|
|
8
9
|
}
|
|
10
|
+
export declare const DEFAULT_MAX_CONTENT_LENGTH_BYTES: number;
|
|
11
|
+
export declare function checkContentLength(mcpServer: McpServer, url: string, timeoutMs: number, dispatcher?: Dispatcher, baseRequestOptions?: RequestInit): Promise<number | null>;
|
|
9
12
|
export declare function fetchAndConvertToMarkdown(mcpServer: McpServer, url: string, timeoutMs?: number, paginationOptions?: PaginationOptions): Promise<string>;
|
|
10
13
|
export {};
|
package/dist/url-reader.js
CHANGED
|
@@ -8,6 +8,8 @@ import { getHttpSecurityConfig } from "./http-security.js";
|
|
|
8
8
|
import { createURLFormatError, createURLSecurityPolicyError, createNetworkError, createServerError, createContentError, createConversionError, createTimeoutError, createEmptyContentWarning, createUnexpectedError } from "./error-handler.js";
|
|
9
9
|
const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
|
|
10
10
|
const MAX_REDIRECTS = 5;
|
|
11
|
+
export const DEFAULT_MAX_CONTENT_LENGTH_BYTES = 5 * 1024 * 1024;
|
|
12
|
+
const HEAD_TIMEOUT_CAP_MS = 3000;
|
|
11
13
|
function isPrivateHostname(hostname) {
|
|
12
14
|
const lower = hostname.toLowerCase().replace(/\.+$/, "");
|
|
13
15
|
return lower === "localhost" || lower.endsWith(".localhost");
|
|
@@ -162,6 +164,53 @@ function applyPaginationOptions(markdownContent, options) {
|
|
|
162
164
|
}
|
|
163
165
|
return result;
|
|
164
166
|
}
|
|
167
|
+
export async function checkContentLength(mcpServer, url, timeoutMs, dispatcher, baseRequestOptions = {}) {
|
|
168
|
+
const controller = new AbortController();
|
|
169
|
+
const timeoutId = setTimeout(() => controller.abort(), Math.min(timeoutMs, HEAD_TIMEOUT_CAP_MS));
|
|
170
|
+
try {
|
|
171
|
+
const requestOptions = {
|
|
172
|
+
...baseRequestOptions,
|
|
173
|
+
method: "HEAD",
|
|
174
|
+
signal: controller.signal,
|
|
175
|
+
redirect: "manual",
|
|
176
|
+
};
|
|
177
|
+
if (dispatcher) {
|
|
178
|
+
requestOptions.dispatcher = dispatcher;
|
|
179
|
+
}
|
|
180
|
+
const response = await undiciFetch(url, requestOptions);
|
|
181
|
+
const contentLength = response.headers.get("content-length");
|
|
182
|
+
if (!contentLength) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
const parsed = parseInt(contentLength, 10);
|
|
186
|
+
return Number.isNaN(parsed) || parsed < 0 ? null : parsed;
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
logMessage(mcpServer, "warning", `HEAD check failed (proceeding with GET): ${error.message}`);
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
clearTimeout(timeoutId);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function getMaxContentLengthBytes(mcpServer) {
|
|
197
|
+
const rawValue = process.env.URL_READ_MAX_CONTENT_LENGTH_BYTES;
|
|
198
|
+
if (rawValue === undefined || rawValue.trim() === "") {
|
|
199
|
+
return DEFAULT_MAX_CONTENT_LENGTH_BYTES;
|
|
200
|
+
}
|
|
201
|
+
const parsed = parseInt(rawValue, 10);
|
|
202
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
203
|
+
logMessage(mcpServer, "warning", `Ignoring invalid URL_READ_MAX_CONTENT_LENGTH_BYTES="${rawValue}". Expected a positive integer; using default ${DEFAULT_MAX_CONTENT_LENGTH_BYTES}.`);
|
|
204
|
+
return DEFAULT_MAX_CONTENT_LENGTH_BYTES;
|
|
205
|
+
}
|
|
206
|
+
return parsed;
|
|
207
|
+
}
|
|
208
|
+
function createContentTooLargeMessage(contentLength, maxBytes) {
|
|
209
|
+
const sizeMB = (contentLength / (1024 * 1024)).toFixed(1);
|
|
210
|
+
const limitMB = (maxBytes / (1024 * 1024)).toFixed(1);
|
|
211
|
+
return (`Content too large: server reports Content-Length of ${sizeMB} MB (limit: ${limitMB} MB). ` +
|
|
212
|
+
`Try using readHeadings or section to fetch only the relevant parts.`);
|
|
213
|
+
}
|
|
165
214
|
export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 10000, paginationOptions = {}) {
|
|
166
215
|
const startTime = Date.now();
|
|
167
216
|
logMessage(mcpServer, "info", `Fetching URL: ${url}`);
|
|
@@ -184,6 +233,7 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
|
|
|
184
233
|
throw createURLFormatError(url);
|
|
185
234
|
}
|
|
186
235
|
assertUrlAllowed(parsedUrl);
|
|
236
|
+
const maxContentLengthBytes = getMaxContentLengthBytes(mcpServer);
|
|
187
237
|
// Create an AbortController instance
|
|
188
238
|
const controller = new AbortController();
|
|
189
239
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -216,6 +266,10 @@ export async function fetchAndConvertToMarkdown(mcpServer, url, timeoutMs = 1000
|
|
|
216
266
|
if (dispatcher) {
|
|
217
267
|
currentRequestOptions.dispatcher = dispatcher;
|
|
218
268
|
}
|
|
269
|
+
const contentLength = await checkContentLength(mcpServer, currentUrl.toString(), timeoutMs, dispatcher, currentRequestOptions);
|
|
270
|
+
if (contentLength !== null && contentLength > maxContentLengthBytes) {
|
|
271
|
+
return createContentTooLargeMessage(contentLength, maxContentLengthBytes);
|
|
272
|
+
}
|
|
219
273
|
// Fetch the URL with the abort signal.
|
|
220
274
|
// Use undici's own fetch so it shares the same internal version as the
|
|
221
275
|
// Agent/ProxyAgent dispatcher — avoids the Node.js bundled-undici vs
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const packageVersion = "1.
|
|
1
|
+
export declare const packageVersion = "1.5.0";
|
package/dist/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const packageVersion = "1.
|
|
1
|
+
export const packageVersion = "1.5.0";
|