@j0hanz/superfetch 1.0.2 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -57
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +6 -10
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +256 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/errors/app-error.d.ts +6 -20
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +7 -18
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +75 -62
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +1 -5
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +4 -12
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts +2 -20
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +22 -47
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/prompts/index.d.ts +0 -3
- package/dist/prompts/index.d.ts.map +1 -1
- package/dist/prompts/index.js +2 -10
- package/dist/prompts/index.js.map +1 -1
- package/dist/resources/cached-content.d.ts +5 -0
- package/dist/resources/cached-content.d.ts.map +1 -0
- package/dist/resources/cached-content.js +93 -0
- package/dist/resources/cached-content.js.map +1 -0
- package/dist/resources/index.d.ts +0 -3
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +40 -5
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts +0 -4
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +11 -6
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +20 -6
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +128 -20
- package/dist/services/cache.js.map +1 -1
- package/dist/services/card-extractor.d.ts +10 -0
- package/dist/services/card-extractor.d.ts.map +1 -0
- package/dist/services/card-extractor.js +194 -0
- package/dist/services/card-extractor.js.map +1 -0
- package/dist/services/extractor.d.ts +12 -19
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +60 -46
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +13 -11
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +143 -54
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +4 -6
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -6
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +57 -27
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts +6 -18
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +104 -79
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +6 -10
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +83 -84
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +6 -12
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +51 -93
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +12 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.js +184 -0
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -0
- package/dist/tools/index.d.ts +0 -4
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +145 -15
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +8 -0
- package/dist/tools/utils/common.d.ts.map +1 -0
- package/dist/tools/utils/common.js +35 -0
- package/dist/tools/utils/common.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts +3 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -0
- package/dist/tools/utils/fetch-pipeline.js +78 -0
- package/dist/tools/utils/fetch-pipeline.js.map +1 -0
- package/dist/tools/utils/index.d.ts +4 -0
- package/dist/tools/utils/index.d.ts.map +1 -0
- package/dist/tools/utils/index.js +3 -0
- package/dist/tools/utils/index.js.map +1 -0
- package/dist/tools/utils/response-builder.d.ts +3 -0
- package/dist/tools/utils/response-builder.d.ts.map +1 -0
- package/dist/tools/utils/response-builder.js +24 -0
- package/dist/tools/utils/response-builder.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +2 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +99 -5
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/types/content.types.d.ts +11 -11
- package/dist/types/content.types.d.ts.map +1 -1
- package/dist/types/index.d.ts +1 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -2
- package/dist/types/index.js.map +1 -1
- package/dist/types/schemas.d.ts +39 -12
- package/dist/types/schemas.d.ts.map +1 -1
- package/dist/utils/concurrency.d.ts +6 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +38 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +32 -0
- package/dist/utils/content-cleaner.d.ts.map +1 -0
- package/dist/utils/content-cleaner.js +238 -0
- package/dist/utils/content-cleaner.js.map +1 -0
- package/dist/utils/language-detector.d.ts +5 -0
- package/dist/utils/language-detector.d.ts.map +1 -0
- package/dist/utils/language-detector.js +50 -0
- package/dist/utils/language-detector.js.map +1 -0
- package/dist/utils/sanitizer.d.ts +0 -10
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +4 -12
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts +1 -15
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +34 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -8
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +17 -31
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +81 -79
package/README.md
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
|
-
# 🚀 superFetch
|
|
1
|
+
# 🚀 superFetch MCP Server
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
<img src="docs/logo.png" alt="SuperFetch MCP Logo" width="200">
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/@j0hanz/superfetch) [](https://nodejs.org/) [](https://www.typescriptlang.org/)
|
|
6
|
+
|
|
7
|
+
## One-Click Install
|
|
8
|
+
|
|
9
|
+
[](https://insiders.vscode.dev/redirect/mcp/install?name=superfetch&inputs=%5B%5D&config=%7B%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22%40j0hanz%2Fsuperfetch%40latest%22%2C%22--stdio%22%5D%7D) [](https://insiders.vscode.dev/redirect/mcp/install?name=superfetch&inputs=%5B%5D&config=%7B%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22%40j0hanz%2Fsuperfetch%40latest%22%2C%22--stdio%22%5D%7D&quality=insiders)
|
|
10
|
+
|
|
11
|
+
[](https://cursor.com/install-mcp?name=superfetch&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBqMGhhbnovc3VwZXJmZXRjaEBsYXRlc3QiLCItLXN0ZGlvIl19)
|
|
4
12
|
|
|
5
13
|
A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
|
|
6
14
|
|
|
7
|
-
|
|
15
|
+
**Version:** 1.0.4
|
|
16
|
+
|
|
17
|
+
[Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
|
|
18
|
+
|
|
19
|
+
> 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
|
|
8
20
|
|
|
9
21
|
---
|
|
10
22
|
|
|
@@ -25,6 +37,43 @@ A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that f
|
|
|
25
37
|
|
|
26
38
|
---
|
|
27
39
|
|
|
40
|
+
## 🎯 How to Choose a Tool
|
|
41
|
+
|
|
42
|
+
Use this guide to select the right tool for your web content extraction needs:
|
|
43
|
+
|
|
44
|
+
### Decision Tree
|
|
45
|
+
|
|
46
|
+
```text
|
|
47
|
+
Need web content for AI?
|
|
48
|
+
├─ Single URL?
|
|
49
|
+
│ ├─ Need structured semantic blocks → fetch-url (JSONL)
|
|
50
|
+
│ ├─ Need readable markdown → fetch-markdown
|
|
51
|
+
│ └─ Need links only → fetch-links
|
|
52
|
+
└─ Multiple URLs?
|
|
53
|
+
└─ Use fetch-urls (batch processing)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Quick Reference Table
|
|
57
|
+
|
|
58
|
+
| Tool | Best For | Output Format | Use When |
|
|
59
|
+
| ---------------- | -------------------------------- | ----------------------- | ------------------------------------------- |
|
|
60
|
+
| `fetch-url` | Single page → structured content | JSONL semantic blocks | AI analysis, RAG pipelines, content parsing |
|
|
61
|
+
| `fetch-markdown` | Single page → readable format | Clean Markdown + TOC | Documentation, human-readable output |
|
|
62
|
+
| `fetch-links` | Link discovery & classification | URL array with types | Sitemap building, finding related pages |
|
|
63
|
+
| `fetch-urls` | Batch processing multiple pages | Multiple JSONL/Markdown | Comparing pages, bulk extraction |
|
|
64
|
+
|
|
65
|
+
### Common Use Cases
|
|
66
|
+
|
|
67
|
+
| Task | Recommended Tool | Why |
|
|
68
|
+
| ------------------------ | ---------------------------------------- | ---------------------------------------------------- |
|
|
69
|
+
| Parse a blog post for AI | `fetch-url` | Returns semantic blocks (headings, paragraphs, code) |
|
|
70
|
+
| Generate documentation | `fetch-markdown` | Clean markdown with optional TOC |
|
|
71
|
+
| Build a sitemap | `fetch-links` | Extracts and classifies all links |
|
|
72
|
+
| Compare multiple docs | `fetch-urls` | Parallel fetching with concurrency control |
|
|
73
|
+
| Extract article for RAG | `fetch-url` + `extractMainContent: true` | Removes ads/nav, keeps main content |
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
28
77
|
## Quick Start
|
|
29
78
|
|
|
30
79
|
Add superFetch to your MCP client configuration — no installation required!
|
|
@@ -76,6 +125,99 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
76
125
|
}
|
|
77
126
|
```
|
|
78
127
|
|
|
128
|
+
### Cursor
|
|
129
|
+
|
|
130
|
+
1. Open Cursor Settings
|
|
131
|
+
2. Go to **Features > MCP Servers**
|
|
132
|
+
3. Click **"+ Add new global MCP server"**
|
|
133
|
+
4. Add this configuration:
|
|
134
|
+
|
|
135
|
+
```json
|
|
136
|
+
{
|
|
137
|
+
"mcpServers": {
|
|
138
|
+
"superFetch": {
|
|
139
|
+
"command": "npx",
|
|
140
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
> **Tip:** On Windows, if you encounter issues, try: `cmd /c "npx -y @j0hanz/superfetch@latest --stdio"`
|
|
147
|
+
|
|
148
|
+
<details>
|
|
149
|
+
<summary><strong>Cline (VS Code Extension)</strong></summary>
|
|
150
|
+
|
|
151
|
+
Open the Cline MCP settings file:
|
|
152
|
+
|
|
153
|
+
**macOS:**
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
code ~/Library/Application\ Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Windows:**
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
code %APPDATA%\Code\User\globalStorage\saoudrizwan.claude-dev\settings\cline_mcp_settings.json
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Add the configuration:
|
|
166
|
+
|
|
167
|
+
```json
|
|
168
|
+
{
|
|
169
|
+
"mcpServers": {
|
|
170
|
+
"superFetch": {
|
|
171
|
+
"command": "npx",
|
|
172
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
173
|
+
"disabled": false,
|
|
174
|
+
"autoApprove": []
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
</details>
|
|
181
|
+
|
|
182
|
+
<details>
|
|
183
|
+
<summary><strong>Windsurf</strong></summary>
|
|
184
|
+
|
|
185
|
+
Add to `./codeium/windsurf/model_config.json`:
|
|
186
|
+
|
|
187
|
+
```json
|
|
188
|
+
{
|
|
189
|
+
"mcpServers": {
|
|
190
|
+
"superFetch": {
|
|
191
|
+
"command": "npx",
|
|
192
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
</details>
|
|
199
|
+
|
|
200
|
+
<details>
|
|
201
|
+
<summary><strong>Claude Desktop (Config File Locations)</strong></summary>
|
|
202
|
+
|
|
203
|
+
**macOS:**
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
# Open config file
|
|
207
|
+
open -e "$HOME/Library/Application Support/Claude/claude_desktop_config.json"
|
|
208
|
+
|
|
209
|
+
# Or with VS Code
|
|
210
|
+
code "$HOME/Library/Application Support/Claude/claude_desktop_config.json"
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
**Windows:**
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
code %APPDATA%\Claude\claude_desktop_config.json
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
</details>
|
|
220
|
+
|
|
79
221
|
---
|
|
80
222
|
|
|
81
223
|
## Installation (Alternative)
|
|
@@ -136,42 +278,173 @@ node dist/index.js --stdio
|
|
|
136
278
|
|
|
137
279
|
### `fetch-url`
|
|
138
280
|
|
|
139
|
-
Fetches a webpage and converts it to AI-readable format.
|
|
281
|
+
Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
|
|
282
|
+
|
|
283
|
+
| Parameter | Type | Default | Description |
|
|
284
|
+
| -------------------- | ------- | ---------- | -------------------------------------------- |
|
|
285
|
+
| `url` | string | _required_ | URL to fetch |
|
|
286
|
+
| `extractMainContent` | boolean | `true` | Use Readability to extract main content |
|
|
287
|
+
| `includeMetadata` | boolean | `true` | Include page metadata (title, description) |
|
|
288
|
+
| `maxContentLength` | number | – | Maximum content length in characters |
|
|
289
|
+
| `customHeaders` | object | – | Custom HTTP headers for the request |
|
|
290
|
+
| `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
|
|
291
|
+
| `retries` | number | `3` | Number of retry attempts (1-10) |
|
|
292
|
+
|
|
293
|
+
**Example Response:**
|
|
140
294
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
295
|
+
```json
|
|
296
|
+
{
|
|
297
|
+
"url": "https://example.com/article",
|
|
298
|
+
"title": "Example Article",
|
|
299
|
+
"fetchedAt": "2025-12-11T10:30:00.000Z",
|
|
300
|
+
"contentBlocks": [
|
|
301
|
+
{
|
|
302
|
+
"type": "metadata",
|
|
303
|
+
"title": "Example Article",
|
|
304
|
+
"description": "A sample article"
|
|
305
|
+
},
|
|
306
|
+
{ "type": "heading", "level": 1, "text": "Introduction" },
|
|
307
|
+
{
|
|
308
|
+
"type": "paragraph",
|
|
309
|
+
"text": "This is the main content of the article..."
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
"type": "code",
|
|
313
|
+
"language": "javascript",
|
|
314
|
+
"content": "console.log('Hello');"
|
|
315
|
+
}
|
|
316
|
+
],
|
|
317
|
+
"cached": false
|
|
318
|
+
}
|
|
319
|
+
```
|
|
149
320
|
|
|
150
321
|
### `fetch-links`
|
|
151
322
|
|
|
152
|
-
Extracts hyperlinks from a webpage with classification.
|
|
323
|
+
Extracts hyperlinks from a webpage with classification. Supports filtering, image links, and link limits.
|
|
324
|
+
|
|
325
|
+
| Parameter | Type | Default | Description |
|
|
326
|
+
| ----------------- | ------- | ---------- | -------------------------------------------- |
|
|
327
|
+
| `url` | string | _required_ | URL to extract links from |
|
|
328
|
+
| `includeExternal` | boolean | `true` | Include external links |
|
|
329
|
+
| `includeInternal` | boolean | `true` | Include internal links |
|
|
330
|
+
| `includeImages` | boolean | `false` | Include image links (img src attributes) |
|
|
331
|
+
| `maxLinks` | number | – | Maximum number of links to return (1-1000) |
|
|
332
|
+
| `filterPattern` | string | – | Regex pattern to filter links (matches href) |
|
|
333
|
+
| `customHeaders` | object | – | Custom HTTP headers for the request |
|
|
334
|
+
| `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
|
|
335
|
+
| `retries` | number | `3` | Number of retry attempts (1-10) |
|
|
336
|
+
|
|
337
|
+
**Example Response:**
|
|
153
338
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
339
|
+
```json
|
|
340
|
+
{
|
|
341
|
+
"url": "https://example.com/",
|
|
342
|
+
"linkCount": 15,
|
|
343
|
+
"links": [
|
|
344
|
+
{
|
|
345
|
+
"href": "https://example.com/about",
|
|
346
|
+
"text": "About Us",
|
|
347
|
+
"type": "internal"
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
"href": "https://github.com/example",
|
|
351
|
+
"text": "GitHub",
|
|
352
|
+
"type": "external"
|
|
353
|
+
},
|
|
354
|
+
{ "href": "https://example.com/logo.png", "text": "", "type": "image" }
|
|
355
|
+
],
|
|
356
|
+
"cached": false,
|
|
357
|
+
"truncated": false
|
|
358
|
+
}
|
|
359
|
+
```
|
|
159
360
|
|
|
160
361
|
### `fetch-markdown`
|
|
161
362
|
|
|
162
|
-
Fetches a webpage and converts it to clean Markdown.
|
|
363
|
+
Fetches a webpage and converts it to clean Markdown with optional table of contents.
|
|
364
|
+
|
|
365
|
+
| Parameter | Type | Default | Description |
|
|
366
|
+
| -------------------- | ------- | ---------- | -------------------------------------------- |
|
|
367
|
+
| `url` | string | _required_ | URL to fetch |
|
|
368
|
+
| `extractMainContent` | boolean | `true` | Extract main content only |
|
|
369
|
+
| `includeMetadata` | boolean | `true` | Include YAML frontmatter |
|
|
370
|
+
| `maxContentLength` | number | – | Maximum content length in characters |
|
|
371
|
+
| `generateToc` | boolean | `false` | Generate table of contents from headings |
|
|
372
|
+
| `customHeaders` | object | – | Custom HTTP headers for the request |
|
|
373
|
+
| `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
|
|
374
|
+
| `retries` | number | `3` | Number of retry attempts (1-10) |
|
|
375
|
+
|
|
376
|
+
**Example Response:**
|
|
377
|
+
|
|
378
|
+
````json
|
|
379
|
+
{
|
|
380
|
+
"url": "https://example.com/docs",
|
|
381
|
+
"title": "Documentation",
|
|
382
|
+
"fetchedAt": "2025-12-11T10:30:00.000Z",
|
|
383
|
+
"markdown": "---\ntitle: Documentation\nsource: \"https://example.com/docs\"\n---\n\n# Getting Started\n\nWelcome to our documentation...\n\n## Installation\n\n```bash\nnpm install example\n```",
|
|
384
|
+
"toc": [
|
|
385
|
+
{ "level": 1, "text": "Getting Started", "slug": "getting-started" },
|
|
386
|
+
{ "level": 2, "text": "Installation", "slug": "installation" }
|
|
387
|
+
],
|
|
388
|
+
"cached": false,
|
|
389
|
+
"truncated": false
|
|
390
|
+
}
|
|
391
|
+
````
|
|
392
|
+
|
|
393
|
+
### `fetch-urls` (Batch)
|
|
394
|
+
|
|
395
|
+
Fetches multiple URLs in parallel with concurrency control. Ideal for comparing content or processing multiple pages efficiently.
|
|
163
396
|
|
|
164
|
-
| Parameter | Type
|
|
165
|
-
| -------------------- |
|
|
166
|
-
| `
|
|
167
|
-
| `extractMainContent` | boolean
|
|
168
|
-
| `includeMetadata` | boolean
|
|
397
|
+
| Parameter | Type | Default | Description |
|
|
398
|
+
| -------------------- | -------- | ---------- | -------------------------------------------- |
|
|
399
|
+
| `urls` | string[] | _required_ | Array of URLs to fetch (1-10 URLs) |
|
|
400
|
+
| `extractMainContent` | boolean | `true` | Use Readability to extract main content |
|
|
401
|
+
| `includeMetadata` | boolean | `true` | Include page metadata |
|
|
402
|
+
| `maxContentLength` | number | – | Maximum content length per URL in characters |
|
|
403
|
+
| `format` | string | `'jsonl'` | Output format: `'jsonl'` or `'markdown'` |
|
|
404
|
+
| `concurrency` | number | `3` | Maximum concurrent requests (1-5) |
|
|
405
|
+
| `continueOnError` | boolean | `true` | Continue processing if some URLs fail |
|
|
406
|
+
| `customHeaders` | object | – | Custom HTTP headers for all requests |
|
|
407
|
+
| `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
|
|
408
|
+
| `retries` | number | `3` | Number of retry attempts (1-10) |
|
|
409
|
+
|
|
410
|
+
**Example Output:**
|
|
411
|
+
|
|
412
|
+
```json
|
|
413
|
+
{
|
|
414
|
+
"results": [
|
|
415
|
+
{
|
|
416
|
+
"url": "https://example.com",
|
|
417
|
+
"success": true,
|
|
418
|
+
"title": "Example",
|
|
419
|
+
"content": "...",
|
|
420
|
+
"cached": false
|
|
421
|
+
},
|
|
422
|
+
{
|
|
423
|
+
"url": "https://example.org",
|
|
424
|
+
"success": true,
|
|
425
|
+
"title": "Example Org",
|
|
426
|
+
"content": "...",
|
|
427
|
+
"cached": false
|
|
428
|
+
}
|
|
429
|
+
],
|
|
430
|
+
"summary": {
|
|
431
|
+
"total": 2,
|
|
432
|
+
"successful": 2,
|
|
433
|
+
"failed": 0,
|
|
434
|
+
"cached": 0,
|
|
435
|
+
"totalContentBlocks": 15
|
|
436
|
+
},
|
|
437
|
+
"fetchedAt": "2024-12-11T10:30:00.000Z"
|
|
438
|
+
}
|
|
439
|
+
```
|
|
169
440
|
|
|
170
441
|
### Resources
|
|
171
442
|
|
|
172
|
-
| URI
|
|
173
|
-
|
|
|
174
|
-
| `superfetch://stats`
|
|
443
|
+
| URI | Description |
|
|
444
|
+
| --------------------- | --------------------------------------------------- |
|
|
445
|
+
| `superfetch://stats` | Server statistics and cache metrics |
|
|
446
|
+
| `superfetch://health` | Real-time server health and dependency status |
|
|
447
|
+
| Dynamic resources | Cached content available via resource subscriptions |
|
|
175
448
|
|
|
176
449
|
### Prompts
|
|
177
450
|
|
|
@@ -227,18 +500,23 @@ Then add to `.vscode/mcp.json`:
|
|
|
227
500
|
|
|
228
501
|
### Environment Variables
|
|
229
502
|
|
|
230
|
-
| Variable
|
|
231
|
-
|
|
|
232
|
-
| `PORT`
|
|
233
|
-
| `HOST`
|
|
234
|
-
| `FETCH_TIMEOUT`
|
|
235
|
-
| `MAX_REDIRECTS`
|
|
236
|
-
| `USER_AGENT`
|
|
237
|
-
| `MAX_CONTENT_LENGTH`
|
|
238
|
-
| `CACHE_ENABLED`
|
|
239
|
-
| `CACHE_TTL`
|
|
240
|
-
| `CACHE_MAX_KEYS`
|
|
241
|
-
| `LOG_LEVEL`
|
|
503
|
+
| Variable | Default | Description |
|
|
504
|
+
| ---------------------- | -------------------- | ------------------------------- |
|
|
505
|
+
| `PORT` | `3000` | HTTP server port |
|
|
506
|
+
| `HOST` | `127.0.0.1` | HTTP server host |
|
|
507
|
+
| `FETCH_TIMEOUT` | `30000` | Request timeout (ms) |
|
|
508
|
+
| `MAX_REDIRECTS` | `5` | Maximum HTTP redirects |
|
|
509
|
+
| `USER_AGENT` | `superFetch-MCP/1.0` | HTTP User-Agent |
|
|
510
|
+
| `MAX_CONTENT_LENGTH` | `10485760` | Max response size (bytes) |
|
|
511
|
+
| `CACHE_ENABLED` | `true` | Enable response caching |
|
|
512
|
+
| `CACHE_TTL` | `3600` | Cache TTL (seconds) |
|
|
513
|
+
| `CACHE_MAX_KEYS` | `100` | Maximum cache entries |
|
|
514
|
+
| `LOG_LEVEL` | `info` | Logging level |
|
|
515
|
+
| `ENABLE_LOGGING` | `true` | Enable/disable logging |
|
|
516
|
+
| `EXTRACT_MAIN_CONTENT` | `true` | Extract main content by default |
|
|
517
|
+
| `INCLUDE_METADATA` | `true` | Include metadata by default |
|
|
518
|
+
| `MAX_BLOCK_LENGTH` | `5000` | Maximum block length |
|
|
519
|
+
| `MIN_PARAGRAPH_LENGTH` | `10` | Minimum paragraph length |
|
|
242
520
|
|
|
243
521
|
---
|
|
244
522
|
|
|
@@ -277,6 +555,19 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
|
|
|
277
555
|
|
|
278
556
|
Default: **100 requests/minute** per IP (configurable)
|
|
279
557
|
|
|
558
|
+
### HTTP Mode Endpoints
|
|
559
|
+
|
|
560
|
+
When running without `--stdio`, the following endpoints are available:
|
|
561
|
+
|
|
562
|
+
| Endpoint | Method | Description |
|
|
563
|
+
| --------- | ------ | --------------------------------------- |
|
|
564
|
+
| `/health` | GET | Health check with uptime and version |
|
|
565
|
+
| `/mcp` | POST | MCP request handling (requires session) |
|
|
566
|
+
| `/mcp` | GET | SSE stream for notifications |
|
|
567
|
+
| `/mcp` | DELETE | Close session |
|
|
568
|
+
|
|
569
|
+
Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
570
|
+
|
|
280
571
|
---
|
|
281
572
|
|
|
282
573
|
## Development
|
|
@@ -291,21 +582,24 @@ Default: **100 requests/minute** per IP (configurable)
|
|
|
291
582
|
| `npm run lint` | Run ESLint |
|
|
292
583
|
| `npm run type-check` | TypeScript type checking |
|
|
293
584
|
| `npm run format` | Format with Prettier |
|
|
294
|
-
| `npm
|
|
585
|
+
| `npm run release` | Create new release |
|
|
586
|
+
| `npm run knip` | Find unused exports/dependencies |
|
|
587
|
+
| `npm run knip:fix` | Auto-fix unused code |
|
|
295
588
|
|
|
296
589
|
### Tech Stack
|
|
297
590
|
|
|
298
|
-
| Category | Technology
|
|
299
|
-
| ------------------ |
|
|
300
|
-
| Runtime | Node.js ≥
|
|
301
|
-
| Language | TypeScript 5.9
|
|
302
|
-
| MCP SDK | @modelcontextprotocol/sdk 1.
|
|
303
|
-
| Content Extraction | @mozilla/readability
|
|
304
|
-
| HTML Parsing | Cheerio, JSDOM
|
|
305
|
-
| Markdown | Turndown
|
|
306
|
-
| HTTP | Express, Axios
|
|
307
|
-
| Caching | node-cache
|
|
308
|
-
| Validation | Zod
|
|
591
|
+
| Category | Technology |
|
|
592
|
+
| ------------------ | --------------------------------- |
|
|
593
|
+
| Runtime | Node.js ≥20.0.0 |
|
|
594
|
+
| Language | TypeScript 5.9 |
|
|
595
|
+
| MCP SDK | @modelcontextprotocol/sdk ^1.24.3 |
|
|
596
|
+
| Content Extraction | @mozilla/readability ^0.6.0 |
|
|
597
|
+
| HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
|
|
598
|
+
| Markdown | Turndown ^7.2.2 |
|
|
599
|
+
| HTTP | Express ^5.2.1, Axios ^1.13.2 |
|
|
600
|
+
| Caching | node-cache ^5.1.2 |
|
|
601
|
+
| Validation | Zod ^3.25.76 |
|
|
602
|
+
| Logging | Winston ^3.19.0 |
|
|
309
603
|
|
|
310
604
|
---
|
|
311
605
|
|
|
@@ -319,9 +613,3 @@ Default: **100 requests/minute** per IP (configurable)
|
|
|
319
613
|
6. Open a Pull Request
|
|
320
614
|
|
|
321
615
|
For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
|
|
322
|
-
|
|
323
|
-
---
|
|
324
|
-
|
|
325
|
-
## License
|
|
326
|
-
|
|
327
|
-
MIT License — see [LICENSE](LICENSE) for details.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCT,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Safely parses an integer from environment variable with validation
|
|
3
|
-
* Environment variables can still be used for deployment overrides (e.g., PORT=8080 npm start)
|
|
4
|
-
*/
|
|
5
1
|
function parseIntEnv(value, defaultValue, min = 0, max = Number.MAX_SAFE_INTEGER) {
|
|
6
2
|
if (!value)
|
|
7
3
|
return defaultValue;
|
|
@@ -15,17 +11,17 @@ export const config = {
|
|
|
15
11
|
name: 'superFetch',
|
|
16
12
|
version: '1.0.0',
|
|
17
13
|
port: parseIntEnv(process.env.PORT, 3000, 1, 65535),
|
|
18
|
-
host: process.env.HOST
|
|
14
|
+
host: process.env.HOST ?? '127.0.0.1',
|
|
19
15
|
},
|
|
20
16
|
fetcher: {
|
|
21
|
-
timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
|
|
17
|
+
timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
|
|
22
18
|
maxRedirects: parseIntEnv(process.env.MAX_REDIRECTS, 5, 0, 20),
|
|
23
|
-
userAgent: process.env.USER_AGENT
|
|
24
|
-
maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
|
|
19
|
+
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
|
|
20
|
+
maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
|
|
25
21
|
},
|
|
26
22
|
cache: {
|
|
27
23
|
enabled: process.env.CACHE_ENABLED !== 'false',
|
|
28
|
-
ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
24
|
+
ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
29
25
|
maxKeys: parseIntEnv(process.env.CACHE_MAX_KEYS, 100, 10, 10000),
|
|
30
26
|
},
|
|
31
27
|
extraction: {
|
|
@@ -35,7 +31,7 @@ export const config = {
|
|
|
35
31
|
minParagraphLength: parseIntEnv(process.env.MIN_PARAGRAPH_LENGTH, 10, 0, 1000),
|
|
36
32
|
},
|
|
37
33
|
logging: {
|
|
38
|
-
level: process.env.LOG_LEVEL
|
|
34
|
+
level: process.env.LOG_LEVEL ?? 'info',
|
|
39
35
|
enabled: process.env.ENABLE_LOGGING !== 'false',
|
|
40
36
|
},
|
|
41
37
|
};
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACpE,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT;KACF;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;CACO,CAAC"}
|