@j0hanz/superfetch 1.0.2 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +345 -57
  2. package/dist/config/index.d.ts.map +1 -1
  3. package/dist/config/index.js +6 -10
  4. package/dist/config/index.js.map +1 -1
  5. package/dist/config/types.d.ts +256 -0
  6. package/dist/config/types.d.ts.map +1 -0
  7. package/dist/config/types.js +2 -0
  8. package/dist/config/types.js.map +1 -0
  9. package/dist/errors/app-error.d.ts +6 -20
  10. package/dist/errors/app-error.d.ts.map +1 -1
  11. package/dist/errors/app-error.js +7 -18
  12. package/dist/errors/app-error.js.map +1 -1
  13. package/dist/index.js +75 -62
  14. package/dist/index.js.map +1 -1
  15. package/dist/middleware/error-handler.d.ts +1 -5
  16. package/dist/middleware/error-handler.d.ts.map +1 -1
  17. package/dist/middleware/error-handler.js +4 -12
  18. package/dist/middleware/error-handler.js.map +1 -1
  19. package/dist/middleware/rate-limiter.d.ts +2 -20
  20. package/dist/middleware/rate-limiter.d.ts.map +1 -1
  21. package/dist/middleware/rate-limiter.js +22 -47
  22. package/dist/middleware/rate-limiter.js.map +1 -1
  23. package/dist/prompts/index.d.ts +0 -3
  24. package/dist/prompts/index.d.ts.map +1 -1
  25. package/dist/prompts/index.js +2 -10
  26. package/dist/prompts/index.js.map +1 -1
  27. package/dist/resources/cached-content.d.ts +5 -0
  28. package/dist/resources/cached-content.d.ts.map +1 -0
  29. package/dist/resources/cached-content.js +93 -0
  30. package/dist/resources/cached-content.js.map +1 -0
  31. package/dist/resources/index.d.ts +0 -3
  32. package/dist/resources/index.d.ts.map +1 -1
  33. package/dist/resources/index.js +40 -5
  34. package/dist/resources/index.js.map +1 -1
  35. package/dist/server.d.ts +0 -4
  36. package/dist/server.d.ts.map +1 -1
  37. package/dist/server.js +11 -6
  38. package/dist/server.js.map +1 -1
  39. package/dist/services/cache.d.ts +20 -6
  40. package/dist/services/cache.d.ts.map +1 -1
  41. package/dist/services/cache.js +128 -20
  42. package/dist/services/cache.js.map +1 -1
  43. package/dist/services/card-extractor.d.ts +10 -0
  44. package/dist/services/card-extractor.d.ts.map +1 -0
  45. package/dist/services/card-extractor.js +194 -0
  46. package/dist/services/card-extractor.js.map +1 -0
  47. package/dist/services/extractor.d.ts +12 -19
  48. package/dist/services/extractor.d.ts.map +1 -1
  49. package/dist/services/extractor.js +60 -46
  50. package/dist/services/extractor.js.map +1 -1
  51. package/dist/services/fetcher.d.ts +13 -11
  52. package/dist/services/fetcher.d.ts.map +1 -1
  53. package/dist/services/fetcher.js +143 -54
  54. package/dist/services/fetcher.js.map +1 -1
  55. package/dist/services/logger.d.ts.map +1 -1
  56. package/dist/services/logger.js +4 -6
  57. package/dist/services/logger.js.map +1 -1
  58. package/dist/services/parser.d.ts +1 -6
  59. package/dist/services/parser.d.ts.map +1 -1
  60. package/dist/services/parser.js +57 -27
  61. package/dist/services/parser.js.map +1 -1
  62. package/dist/tools/handlers/fetch-links.tool.d.ts +6 -18
  63. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  64. package/dist/tools/handlers/fetch-links.tool.js +104 -79
  65. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  66. package/dist/tools/handlers/fetch-markdown.tool.d.ts +6 -10
  67. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  68. package/dist/tools/handlers/fetch-markdown.tool.js +83 -84
  69. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  70. package/dist/tools/handlers/fetch-url.tool.d.ts +6 -12
  71. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  72. package/dist/tools/handlers/fetch-url.tool.js +51 -93
  73. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  74. package/dist/tools/handlers/fetch-urls.tool.d.ts +12 -0
  75. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -0
  76. package/dist/tools/handlers/fetch-urls.tool.js +184 -0
  77. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -0
  78. package/dist/tools/index.d.ts +0 -4
  79. package/dist/tools/index.d.ts.map +1 -1
  80. package/dist/tools/index.js +145 -15
  81. package/dist/tools/index.js.map +1 -1
  82. package/dist/tools/utils/common.d.ts +8 -0
  83. package/dist/tools/utils/common.d.ts.map +1 -0
  84. package/dist/tools/utils/common.js +35 -0
  85. package/dist/tools/utils/common.js.map +1 -0
  86. package/dist/tools/utils/fetch-pipeline.d.ts +3 -0
  87. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -0
  88. package/dist/tools/utils/fetch-pipeline.js +78 -0
  89. package/dist/tools/utils/fetch-pipeline.js.map +1 -0
  90. package/dist/tools/utils/index.d.ts +4 -0
  91. package/dist/tools/utils/index.d.ts.map +1 -0
  92. package/dist/tools/utils/index.js +3 -0
  93. package/dist/tools/utils/index.js.map +1 -0
  94. package/dist/tools/utils/response-builder.d.ts +3 -0
  95. package/dist/tools/utils/response-builder.d.ts.map +1 -0
  96. package/dist/tools/utils/response-builder.js +24 -0
  97. package/dist/tools/utils/response-builder.js.map +1 -0
  98. package/dist/transformers/jsonl.transformer.d.ts +1 -1
  99. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  100. package/dist/transformers/jsonl.transformer.js +2 -1
  101. package/dist/transformers/jsonl.transformer.js.map +1 -1
  102. package/dist/transformers/markdown.transformer.d.ts +1 -1
  103. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  104. package/dist/transformers/markdown.transformer.js +99 -5
  105. package/dist/transformers/markdown.transformer.js.map +1 -1
  106. package/dist/types/content.types.d.ts +11 -11
  107. package/dist/types/content.types.d.ts.map +1 -1
  108. package/dist/types/index.d.ts +1 -2
  109. package/dist/types/index.d.ts.map +1 -1
  110. package/dist/types/index.js +1 -2
  111. package/dist/types/index.js.map +1 -1
  112. package/dist/types/schemas.d.ts +39 -12
  113. package/dist/types/schemas.d.ts.map +1 -1
  114. package/dist/utils/concurrency.d.ts +6 -0
  115. package/dist/utils/concurrency.d.ts.map +1 -0
  116. package/dist/utils/concurrency.js +38 -0
  117. package/dist/utils/concurrency.js.map +1 -0
  118. package/dist/utils/content-cleaner.d.ts +32 -0
  119. package/dist/utils/content-cleaner.d.ts.map +1 -0
  120. package/dist/utils/content-cleaner.js +238 -0
  121. package/dist/utils/content-cleaner.js.map +1 -0
  122. package/dist/utils/language-detector.d.ts +5 -0
  123. package/dist/utils/language-detector.d.ts.map +1 -0
  124. package/dist/utils/language-detector.js +50 -0
  125. package/dist/utils/language-detector.js.map +1 -0
  126. package/dist/utils/sanitizer.d.ts +0 -10
  127. package/dist/utils/sanitizer.d.ts.map +1 -1
  128. package/dist/utils/sanitizer.js +4 -12
  129. package/dist/utils/sanitizer.js.map +1 -1
  130. package/dist/utils/tool-error-handler.d.ts +1 -15
  131. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  132. package/dist/utils/tool-error-handler.js +34 -6
  133. package/dist/utils/tool-error-handler.js.map +1 -1
  134. package/dist/utils/url-validator.d.ts +0 -8
  135. package/dist/utils/url-validator.d.ts.map +1 -1
  136. package/dist/utils/url-validator.js +17 -31
  137. package/dist/utils/url-validator.js.map +1 -1
  138. package/package.json +81 -79
package/README.md CHANGED
@@ -1,10 +1,22 @@
1
- # 🚀 superFetch
1
+ # 🚀 superFetch MCP Server
2
2
 
3
- [![npm version](https://img.shields.io/npm/v/@j0hanz/superfetch.svg)](https://www.npmjs.com/package/@j0hanz/superfetch)[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)[![Node.js](https://img.shields.io/badge/Node.js-≥18.0.0-339933?logo=nodedotjs&logoColor=white)](https://nodejs.org/)[![TypeScript](https://img.shields.io/badge/TypeScript-5.9-3178C6?logo=typescript&logoColor=white)](https://www.typescriptlang.org/)[![MCP](https://img.shields.io/badge/MCP-1.0.4-8B5CF6?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZmlsbD0id2hpdGUiIGQ9Ik0xMiAyQzYuNDggMiAyIDYuNDggMiAxMnM0LjQ4IDEwIDEwIDEwIDEwLTQuNDggMTAtMTBTMTcuNTIgMiAxMiAyem0wIDE4Yy00LjQyIDAtOC0zLjU4LTgtOHMzLjU4LTggOC04IDggMy41OCA4IDgtMy41OCA4LTggOHoiLz48L3N2Zz4=)](https://modelcontextprotocol.io/)
3
+ <img src="docs/logo.png" alt="SuperFetch MCP Logo" width="200">
4
+
5
+ [![npm version](https://img.shields.io/npm/v/@j0hanz/superfetch.svg)](https://www.npmjs.com/package/@j0hanz/superfetch) [![Node.js](https://img.shields.io/badge/Node.js-≥20.0.0-339933?logo=nodedotjs&logoColor=white)](https://nodejs.org/) [![TypeScript](https://img.shields.io/badge/TypeScript-5.9-3178C6?logo=typescript&logoColor=white)](https://www.typescriptlang.org/)
6
+
7
+ ## One-Click Install
8
+
9
+ [![Install with NPX in VS Code](https://img.shields.io/badge/VS_Code-Install-0098FF?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=superfetch&inputs=%5B%5D&config=%7B%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22%40j0hanz%2Fsuperfetch%40latest%22%2C%22--stdio%22%5D%7D) [![Install with NPX in VS Code Insiders](https://img.shields.io/badge/VS_Code_Insiders-Install-24bfa5?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=superfetch&inputs=%5B%5D&config=%7B%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22%40j0hanz%2Fsuperfetch%40latest%22%2C%22--stdio%22%5D%7D&quality=insiders)
10
+
11
+ [![Install in Cursor](https://cursor.com/deeplink/mcp-install-dark.svg)](https://cursor.com/install-mcp?name=superfetch&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBqMGhhbnovc3VwZXJmZXRjaEBsYXRlc3QiLCItLXN0ZGlvIl19)
4
12
 
5
13
  A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
6
14
 
7
- [Quick Start](#quick-start) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
15
+ **Version:** 1.0.4
16
+
17
+ [Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
18
+
19
+ > 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
8
20
 
9
21
  ---
10
22
 
@@ -25,6 +37,43 @@ A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that f
25
37
 
26
38
  ---
27
39
 
40
+ ## 🎯 How to Choose a Tool
41
+
42
+ Use this guide to select the right tool for your web content extraction needs:
43
+
44
+ ### Decision Tree
45
+
46
+ ```text
47
+ Need web content for AI?
48
+ ├─ Single URL?
49
+ │ ├─ Need structured semantic blocks → fetch-url (JSONL)
50
+ │ ├─ Need readable markdown → fetch-markdown
51
+ │ └─ Need links only → fetch-links
52
+ └─ Multiple URLs?
53
+ └─ Use fetch-urls (batch processing)
54
+ ```
55
+
56
+ ### Quick Reference Table
57
+
58
+ | Tool | Best For | Output Format | Use When |
59
+ | ---------------- | -------------------------------- | ----------------------- | ------------------------------------------- |
60
+ | `fetch-url` | Single page → structured content | JSONL semantic blocks | AI analysis, RAG pipelines, content parsing |
61
+ | `fetch-markdown` | Single page → readable format | Clean Markdown + TOC | Documentation, human-readable output |
62
+ | `fetch-links` | Link discovery & classification | URL array with types | Sitemap building, finding related pages |
63
+ | `fetch-urls` | Batch processing multiple pages | Multiple JSONL/Markdown | Comparing pages, bulk extraction |
64
+
65
+ ### Common Use Cases
66
+
67
+ | Task | Recommended Tool | Why |
68
+ | ------------------------ | ---------------------------------------- | ---------------------------------------------------- |
69
+ | Parse a blog post for AI | `fetch-url` | Returns semantic blocks (headings, paragraphs, code) |
70
+ | Generate documentation | `fetch-markdown` | Clean markdown with optional TOC |
71
+ | Build a sitemap | `fetch-links` | Extracts and classifies all links |
72
+ | Compare multiple docs | `fetch-urls` | Parallel fetching with concurrency control |
73
+ | Extract article for RAG | `fetch-url` + `extractMainContent: true` | Removes ads/nav, keeps main content |
74
+
75
+ ---
76
+
28
77
  ## Quick Start
29
78
 
30
79
  Add superFetch to your MCP client configuration — no installation required!
@@ -76,6 +125,99 @@ Add to `.vscode/mcp.json` in your workspace:
76
125
  }
77
126
  ```
78
127
 
128
+ ### Cursor
129
+
130
+ 1. Open Cursor Settings
131
+ 2. Go to **Features > MCP Servers**
132
+ 3. Click **"+ Add new global MCP server"**
133
+ 4. Add this configuration:
134
+
135
+ ```json
136
+ {
137
+ "mcpServers": {
138
+ "superFetch": {
139
+ "command": "npx",
140
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
141
+ }
142
+ }
143
+ }
144
+ ```
145
+
146
+ > **Tip:** On Windows, if you encounter issues, try: `cmd /c "npx -y @j0hanz/superfetch@latest --stdio"`
147
+
148
+ <details>
149
+ <summary><strong>Cline (VS Code Extension)</strong></summary>
150
+
151
+ Open the Cline MCP settings file:
152
+
153
+ **macOS:**
154
+
155
+ ```bash
156
+ code ~/Library/Application\ Support/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json
157
+ ```
158
+
159
+ **Windows:**
160
+
161
+ ```bash
162
+ code %APPDATA%\Code\User\globalStorage\saoudrizwan.claude-dev\settings\cline_mcp_settings.json
163
+ ```
164
+
165
+ Add the configuration:
166
+
167
+ ```json
168
+ {
169
+ "mcpServers": {
170
+ "superFetch": {
171
+ "command": "npx",
172
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
173
+ "disabled": false,
174
+ "autoApprove": []
175
+ }
176
+ }
177
+ }
178
+ ```
179
+
180
+ </details>
181
+
182
+ <details>
183
+ <summary><strong>Windsurf</strong></summary>
184
+
185
+ Add to `./codeium/windsurf/model_config.json`:
186
+
187
+ ```json
188
+ {
189
+ "mcpServers": {
190
+ "superFetch": {
191
+ "command": "npx",
192
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
193
+ }
194
+ }
195
+ }
196
+ ```
197
+
198
+ </details>
199
+
200
+ <details>
201
+ <summary><strong>Claude Desktop (Config File Locations)</strong></summary>
202
+
203
+ **macOS:**
204
+
205
+ ```bash
206
+ # Open config file
207
+ open -e "$HOME/Library/Application Support/Claude/claude_desktop_config.json"
208
+
209
+ # Or with VS Code
210
+ code "$HOME/Library/Application Support/Claude/claude_desktop_config.json"
211
+ ```
212
+
213
+ **Windows:**
214
+
215
+ ```bash
216
+ code %APPDATA%\Claude\claude_desktop_config.json
217
+ ```
218
+
219
+ </details>
220
+
79
221
  ---
80
222
 
81
223
  ## Installation (Alternative)
@@ -136,42 +278,173 @@ node dist/index.js --stdio
136
278
 
137
279
  ### `fetch-url`
138
280
 
139
- Fetches a webpage and converts it to AI-readable format.
281
+ Fetches a webpage and converts it to AI-readable JSONL format with semantic content blocks.
282
+
283
+ | Parameter | Type | Default | Description |
284
+ | -------------------- | ------- | ---------- | -------------------------------------------- |
285
+ | `url` | string | _required_ | URL to fetch |
286
+ | `extractMainContent` | boolean | `true` | Use Readability to extract main content |
287
+ | `includeMetadata` | boolean | `true` | Include page metadata (title, description) |
288
+ | `maxContentLength` | number | – | Maximum content length in characters |
289
+ | `customHeaders` | object | – | Custom HTTP headers for the request |
290
+ | `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
291
+ | `retries` | number | `3` | Number of retry attempts (1-10) |
292
+
293
+ **Example Response:**
140
294
 
141
- | Parameter | Type | Default | Description |
142
- | -------------------- | ------------------------- | ---------- | -------------------------- |
143
- | `url` | string | _required_ | URL to fetch |
144
- | `format` | `'jsonl'` \| `'markdown'` | `'jsonl'` | Output format |
145
- | `extractMainContent` | boolean | `true` | Use Readability extraction |
146
- | `includeMetadata` | boolean | `true` | Include page metadata |
147
- | `maxContentLength` | number | – | Max content length (chars) |
148
- | `customHeaders` | object | – | Custom HTTP headers |
295
+ ```json
296
+ {
297
+ "url": "https://example.com/article",
298
+ "title": "Example Article",
299
+ "fetchedAt": "2025-12-11T10:30:00.000Z",
300
+ "contentBlocks": [
301
+ {
302
+ "type": "metadata",
303
+ "title": "Example Article",
304
+ "description": "A sample article"
305
+ },
306
+ { "type": "heading", "level": 1, "text": "Introduction" },
307
+ {
308
+ "type": "paragraph",
309
+ "text": "This is the main content of the article..."
310
+ },
311
+ {
312
+ "type": "code",
313
+ "language": "javascript",
314
+ "content": "console.log('Hello');"
315
+ }
316
+ ],
317
+ "cached": false
318
+ }
319
+ ```
149
320
 
150
321
  ### `fetch-links`
151
322
 
152
- Extracts hyperlinks from a webpage with classification.
323
+ Extracts hyperlinks from a webpage with classification. Supports filtering, image links, and link limits.
324
+
325
+ | Parameter | Type | Default | Description |
326
+ | ----------------- | ------- | ---------- | -------------------------------------------- |
327
+ | `url` | string | _required_ | URL to extract links from |
328
+ | `includeExternal` | boolean | `true` | Include external links |
329
+ | `includeInternal` | boolean | `true` | Include internal links |
330
+ | `includeImages` | boolean | `false` | Include image links (img src attributes) |
331
+ | `maxLinks` | number | – | Maximum number of links to return (1-1000) |
332
+ | `filterPattern` | string | – | Regex pattern to filter links (matches href) |
333
+ | `customHeaders` | object | – | Custom HTTP headers for the request |
334
+ | `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
335
+ | `retries` | number | `3` | Number of retry attempts (1-10) |
336
+
337
+ **Example Response:**
153
338
 
154
- | Parameter | Type | Default | Description |
155
- | ----------------- | ------- | ---------- | ------------------------- |
156
- | `url` | string | _required_ | URL to extract links from |
157
- | `includeExternal` | boolean | `true` | Include external links |
158
- | `includeInternal` | boolean | `true` | Include internal links |
339
+ ```json
340
+ {
341
+ "url": "https://example.com/",
342
+ "linkCount": 15,
343
+ "links": [
344
+ {
345
+ "href": "https://example.com/about",
346
+ "text": "About Us",
347
+ "type": "internal"
348
+ },
349
+ {
350
+ "href": "https://github.com/example",
351
+ "text": "GitHub",
352
+ "type": "external"
353
+ },
354
+ { "href": "https://example.com/logo.png", "text": "", "type": "image" }
355
+ ],
356
+ "cached": false,
357
+ "truncated": false
358
+ }
359
+ ```
159
360
 
160
361
  ### `fetch-markdown`
161
362
 
162
- Fetches a webpage and converts it to clean Markdown.
363
+ Fetches a webpage and converts it to clean Markdown with optional table of contents.
364
+
365
+ | Parameter | Type | Default | Description |
366
+ | -------------------- | ------- | ---------- | -------------------------------------------- |
367
+ | `url` | string | _required_ | URL to fetch |
368
+ | `extractMainContent` | boolean | `true` | Extract main content only |
369
+ | `includeMetadata` | boolean | `true` | Include YAML frontmatter |
370
+ | `maxContentLength` | number | – | Maximum content length in characters |
371
+ | `generateToc` | boolean | `false` | Generate table of contents from headings |
372
+ | `customHeaders` | object | – | Custom HTTP headers for the request |
373
+ | `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
374
+ | `retries` | number | `3` | Number of retry attempts (1-10) |
375
+
376
+ **Example Response:**
377
+
378
+ ````json
379
+ {
380
+ "url": "https://example.com/docs",
381
+ "title": "Documentation",
382
+ "fetchedAt": "2025-12-11T10:30:00.000Z",
383
+ "markdown": "---\ntitle: Documentation\nsource: \"https://example.com/docs\"\n---\n\n# Getting Started\n\nWelcome to our documentation...\n\n## Installation\n\n```bash\nnpm install example\n```",
384
+ "toc": [
385
+ { "level": 1, "text": "Getting Started", "slug": "getting-started" },
386
+ { "level": 2, "text": "Installation", "slug": "installation" }
387
+ ],
388
+ "cached": false,
389
+ "truncated": false
390
+ }
391
+ ````
392
+
393
+ ### `fetch-urls` (Batch)
394
+
395
+ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing content or processing multiple pages efficiently.
163
396
 
164
- | Parameter | Type | Default | Description |
165
- | -------------------- | ------- | ---------- | ------------------------- |
166
- | `url` | string | _required_ | URL to fetch |
167
- | `extractMainContent` | boolean | `true` | Extract main content only |
168
- | `includeMetadata` | boolean | `true` | Include YAML frontmatter |
397
+ | Parameter | Type | Default | Description |
398
+ | -------------------- | -------- | ---------- | -------------------------------------------- |
399
+ | `urls` | string[] | _required_ | Array of URLs to fetch (1-10 URLs) |
400
+ | `extractMainContent` | boolean | `true` | Use Readability to extract main content |
401
+ | `includeMetadata` | boolean | `true` | Include page metadata |
402
+ | `maxContentLength` | number | – | Maximum content length per URL in characters |
403
+ | `format` | string | `'jsonl'` | Output format: `'jsonl'` or `'markdown'` |
404
+ | `concurrency` | number | `3` | Maximum concurrent requests (1-5) |
405
+ | `continueOnError` | boolean | `true` | Continue processing if some URLs fail |
406
+ | `customHeaders` | object | – | Custom HTTP headers for all requests |
407
+ | `timeout` | number | `30000` | Request timeout in milliseconds (1000-60000) |
408
+ | `retries` | number | `3` | Number of retry attempts (1-10) |
409
+
410
+ **Example Output:**
411
+
412
+ ```json
413
+ {
414
+ "results": [
415
+ {
416
+ "url": "https://example.com",
417
+ "success": true,
418
+ "title": "Example",
419
+ "content": "...",
420
+ "cached": false
421
+ },
422
+ {
423
+ "url": "https://example.org",
424
+ "success": true,
425
+ "title": "Example Org",
426
+ "content": "...",
427
+ "cached": false
428
+ }
429
+ ],
430
+ "summary": {
431
+ "total": 2,
432
+ "successful": 2,
433
+ "failed": 0,
434
+ "cached": 0,
435
+ "totalContentBlocks": 15
436
+ },
437
+ "fetchedAt": "2024-12-11T10:30:00.000Z"
438
+ }
439
+ ```
169
440
 
170
441
  ### Resources
171
442
 
172
- | URI | Description |
173
- | -------------------- | ----------------------------------- |
174
- | `superfetch://stats` | Server statistics and cache metrics |
443
+ | URI | Description |
444
+ | --------------------- | --------------------------------------------------- |
445
+ | `superfetch://stats` | Server statistics and cache metrics |
446
+ | `superfetch://health` | Real-time server health and dependency status |
447
+ | Dynamic resources | Cached content available via resource subscriptions |
175
448
 
176
449
  ### Prompts
177
450
 
@@ -227,18 +500,23 @@ Then add to `.vscode/mcp.json`:
227
500
 
228
501
  ### Environment Variables
229
502
 
230
- | Variable | Default | Description |
231
- | -------------------- | -------------------- | ------------------------- |
232
- | `PORT` | `3000` | HTTP server port |
233
- | `HOST` | `127.0.0.1` | HTTP server host |
234
- | `FETCH_TIMEOUT` | `30000` | Request timeout (ms) |
235
- | `MAX_REDIRECTS` | `5` | Maximum HTTP redirects |
236
- | `USER_AGENT` | `superFetch-MCP/1.0` | HTTP User-Agent |
237
- | `MAX_CONTENT_LENGTH` | `10485760` | Max response size (bytes) |
238
- | `CACHE_ENABLED` | `true` | Enable response caching |
239
- | `CACHE_TTL` | `3600` | Cache TTL (seconds) |
240
- | `CACHE_MAX_KEYS` | `100` | Maximum cache entries |
241
- | `LOG_LEVEL` | `info` | Logging level |
503
+ | Variable | Default | Description |
504
+ | ---------------------- | -------------------- | ------------------------------- |
505
+ | `PORT` | `3000` | HTTP server port |
506
+ | `HOST` | `127.0.0.1` | HTTP server host |
507
+ | `FETCH_TIMEOUT` | `30000` | Request timeout (ms) |
508
+ | `MAX_REDIRECTS` | `5` | Maximum HTTP redirects |
509
+ | `USER_AGENT` | `superFetch-MCP/1.0` | HTTP User-Agent |
510
+ | `MAX_CONTENT_LENGTH` | `10485760` | Max response size (bytes) |
511
+ | `CACHE_ENABLED` | `true` | Enable response caching |
512
+ | `CACHE_TTL` | `3600` | Cache TTL (seconds) |
513
+ | `CACHE_MAX_KEYS` | `100` | Maximum cache entries |
514
+ | `LOG_LEVEL` | `info` | Logging level |
515
+ | `ENABLE_LOGGING` | `true` | Enable/disable logging |
516
+ | `EXTRACT_MAIN_CONTENT` | `true` | Extract main content by default |
517
+ | `INCLUDE_METADATA` | `true` | Include metadata by default |
518
+ | `MAX_BLOCK_LENGTH` | `5000` | Maximum block length |
519
+ | `MIN_PARAGRAPH_LENGTH` | `10` | Minimum paragraph length |
242
520
 
243
521
  ---
244
522
 
@@ -277,6 +555,19 @@ Blocked headers: `host`, `authorization`, `cookie`, `x-forwarded-for`, `x-real-i
277
555
 
278
556
  Default: **100 requests/minute** per IP (configurable)
279
557
 
558
+ ### HTTP Mode Endpoints
559
+
560
+ When running without `--stdio`, the following endpoints are available:
561
+
562
+ | Endpoint | Method | Description |
563
+ | --------- | ------ | --------------------------------------- |
564
+ | `/health` | GET | Health check with uptime and version |
565
+ | `/mcp` | POST | MCP request handling (requires session) |
566
+ | `/mcp` | GET | SSE stream for notifications |
567
+ | `/mcp` | DELETE | Close session |
568
+
569
+ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
570
+
280
571
  ---
281
572
 
282
573
  ## Development
@@ -291,21 +582,24 @@ Default: **100 requests/minute** per IP (configurable)
291
582
  | `npm run lint` | Run ESLint |
292
583
  | `npm run type-check` | TypeScript type checking |
293
584
  | `npm run format` | Format with Prettier |
294
- | `npm test` | Run tests |
585
+ | `npm run release` | Create new release |
586
+ | `npm run knip` | Find unused exports/dependencies |
587
+ | `npm run knip:fix` | Auto-fix unused code |
295
588
 
296
589
  ### Tech Stack
297
590
 
298
- | Category | Technology |
299
- | ------------------ | ------------------------------- |
300
- | Runtime | Node.js ≥18 |
301
- | Language | TypeScript 5.9 |
302
- | MCP SDK | @modelcontextprotocol/sdk 1.0.4 |
303
- | Content Extraction | @mozilla/readability |
304
- | HTML Parsing | Cheerio, JSDOM |
305
- | Markdown | Turndown |
306
- | HTTP | Express, Axios |
307
- | Caching | node-cache |
308
- | Validation | Zod |
591
+ | Category | Technology |
592
+ | ------------------ | --------------------------------- |
593
+ | Runtime | Node.js ≥20.0.0 |
594
+ | Language | TypeScript 5.9 |
595
+ | MCP SDK | @modelcontextprotocol/sdk ^1.24.3 |
596
+ | Content Extraction | @mozilla/readability ^0.6.0 |
597
+ | HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
598
+ | Markdown | Turndown ^7.2.2 |
599
+ | HTTP | Express ^5.2.1, Axios ^1.13.2 |
600
+ | Caching | node-cache ^5.1.2 |
601
+ | Validation | Zod ^3.25.76 |
602
+ | Logging | Winston ^3.19.0 |
309
603
 
310
604
  ---
311
605
 
@@ -319,9 +613,3 @@ Default: **100 requests/minute** per IP (configurable)
319
613
  6. Open a Pull Request
320
614
 
321
615
  For examples of other MCP servers, see: [github.com/modelcontextprotocol/servers](https://github.com/modelcontextprotocol/servers)
322
-
323
- ---
324
-
325
- ## License
326
-
327
- MIT License — see [LICENSE](LICENSE) for details.
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCT,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCT,CAAC"}
@@ -1,7 +1,3 @@
1
- /**
2
- * Safely parses an integer from environment variable with validation
3
- * Environment variables can still be used for deployment overrides (e.g., PORT=8080 npm start)
4
- */
5
1
  function parseIntEnv(value, defaultValue, min = 0, max = Number.MAX_SAFE_INTEGER) {
6
2
  if (!value)
7
3
  return defaultValue;
@@ -15,17 +11,17 @@ export const config = {
15
11
  name: 'superFetch',
16
12
  version: '1.0.0',
17
13
  port: parseIntEnv(process.env.PORT, 3000, 1, 65535),
18
- host: process.env.HOST || '127.0.0.1',
14
+ host: process.env.HOST ?? '127.0.0.1',
19
15
  },
20
16
  fetcher: {
21
- timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000), // 1s-120s
17
+ timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
22
18
  maxRedirects: parseIntEnv(process.env.MAX_REDIRECTS, 5, 0, 20),
23
- userAgent: process.env.USER_AGENT || 'superFetch-MCP/1.0',
24
- maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800), // 1KB-50MB
19
+ userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
20
+ maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
25
21
  },
26
22
  cache: {
27
23
  enabled: process.env.CACHE_ENABLED !== 'false',
28
- ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400), // 1min-24hr
24
+ ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
29
25
  maxKeys: parseIntEnv(process.env.CACHE_MAX_KEYS, 100, 10, 10000),
30
26
  },
31
27
  extraction: {
@@ -35,7 +31,7 @@ export const config = {
35
31
  minParagraphLength: parseIntEnv(process.env.MIN_PARAGRAPH_LENGTH, 10, 0, 1000),
36
32
  },
37
33
  logging: {
38
- level: process.env.LOG_LEVEL || 'info',
34
+ level: process.env.LOG_LEVEL ?? 'info',
39
35
  enabled: process.env.ENABLE_LOGGING !== 'false',
40
36
  },
41
37
  };
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,UAAU;QAChF,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT,EAAE,WAAW;KACf;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,YAAY;QACtE,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;CACO,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACpE,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT;KACF;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;CACO,CAAC"}