@j0hanz/superfetch 1.0.6 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +228 -36
- package/dist/config/index.d.ts +10 -5
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +73 -19
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +98 -57
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -28
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +10 -51
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +10 -55
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +12 -14
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +0 -8
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/parsers/base-html-element-parser.d.ts +43 -0
- package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
- package/dist/parsers/base-html-element-parser.js +59 -0
- package/dist/parsers/base-html-element-parser.js.map +1 -0
- package/dist/parsers/heading-element-parser.d.ts +14 -0
- package/dist/parsers/heading-element-parser.d.ts.map +1 -0
- package/dist/parsers/heading-element-parser.js +26 -0
- package/dist/parsers/heading-element-parser.js.map +1 -0
- package/dist/parsers/image-element-parser.d.ts +16 -0
- package/dist/parsers/image-element-parser.d.ts.map +1 -0
- package/dist/parsers/image-element-parser.js +33 -0
- package/dist/parsers/image-element-parser.js.map +1 -0
- package/dist/parsers/link-element-parser.d.ts +15 -0
- package/dist/parsers/link-element-parser.d.ts.map +1 -0
- package/dist/parsers/link-element-parser.js +28 -0
- package/dist/parsers/link-element-parser.js.map +1 -0
- package/dist/parsers/open-graph-parser.d.ts +17 -0
- package/dist/parsers/open-graph-parser.d.ts.map +1 -0
- package/dist/parsers/open-graph-parser.js +41 -0
- package/dist/parsers/open-graph-parser.js.map +1 -0
- package/dist/parsers/schema-org-parser.d.ts +17 -0
- package/dist/parsers/schema-org-parser.d.ts.map +1 -0
- package/dist/parsers/schema-org-parser.js +32 -0
- package/dist/parsers/schema-org-parser.js.map +1 -0
- package/dist/parsers/standard-meta-parser.d.ts +18 -0
- package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
- package/dist/parsers/standard-meta-parser.js +32 -0
- package/dist/parsers/standard-meta-parser.js.map +1 -0
- package/dist/parsers/twitter-card-parser.d.ts +17 -0
- package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
- package/dist/parsers/twitter-card-parser.js +41 -0
- package/dist/parsers/twitter-card-parser.js.map +1 -0
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -9
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +8 -8
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +10 -10
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +0 -28
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +10 -173
- package/dist/services/cache.js.map +1 -1
- package/dist/services/extractor.d.ts +1 -11
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +86 -84
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +2 -13
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +195 -211
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +5 -4
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +27 -42
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +35 -26
- package/dist/services/parser.js.map +1 -1
- package/dist/services/session-manager.d.ts +18 -0
- package/dist/services/session-manager.d.ts.map +1 -0
- package/dist/services/session-manager.js +73 -0
- package/dist/services/session-manager.js.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.js +32 -0
- package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -1
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +15 -20
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +124 -105
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +0 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +6 -7
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +8 -8
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +47 -79
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +15 -10
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +58 -62
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -5
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +19 -19
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts +0 -25
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +12 -187
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +2 -0
- package/dist/utils/html-truncator.d.ts.map +1 -0
- package/dist/utils/html-truncator.js +14 -0
- package/dist/utils/html-truncator.js.map +1 -0
- package/dist/utils/language-detector.d.ts +0 -3
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/language-detector.js +0 -11
- package/dist/utils/language-detector.js.map +1 -1
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +7 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +15 -42
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -6
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +12 -81
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +5 -6
package/README.md
CHANGED
|
@@ -12,8 +12,6 @@
|
|
|
12
12
|
|
|
13
13
|
A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
|
|
14
14
|
|
|
15
|
-
**Version:** 1.0.5
|
|
16
|
-
|
|
17
15
|
[Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
|
|
18
16
|
|
|
19
17
|
> 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
|
|
@@ -108,7 +106,9 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
108
106
|
}
|
|
109
107
|
```
|
|
110
108
|
|
|
111
|
-
### With
|
|
109
|
+
### With Custom Configuration
|
|
110
|
+
|
|
111
|
+
Configure SuperFetch behavior by adding environment variables to the `env` property:
|
|
112
112
|
|
|
113
113
|
```json
|
|
114
114
|
{
|
|
@@ -118,13 +118,16 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
118
118
|
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
119
119
|
"env": {
|
|
120
120
|
"CACHE_TTL": "7200",
|
|
121
|
-
"LOG_LEVEL": "debug"
|
|
121
|
+
"LOG_LEVEL": "debug",
|
|
122
|
+
"FETCH_TIMEOUT": "60000"
|
|
122
123
|
}
|
|
123
124
|
}
|
|
124
125
|
}
|
|
125
126
|
}
|
|
126
127
|
```
|
|
127
128
|
|
|
129
|
+
See [Configuration](#configuration) section below for all available options and presets.
|
|
130
|
+
|
|
128
131
|
### Cursor
|
|
129
132
|
|
|
130
133
|
1. Open Cursor Settings
|
|
@@ -145,6 +148,34 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
145
148
|
|
|
146
149
|
> **Tip:** On Windows, if you encounter issues, try: `cmd /c "npx -y @j0hanz/superfetch@latest --stdio"`
|
|
147
150
|
|
|
151
|
+
<details>
|
|
152
|
+
<summary><strong>Codex IDE</strong></summary>
|
|
153
|
+
|
|
154
|
+
Add to your `~/.codex/config.toml` file:
|
|
155
|
+
|
|
156
|
+
**Basic Configuration:**
|
|
157
|
+
|
|
158
|
+
```toml
|
|
159
|
+
[mcp_servers.superfetch]
|
|
160
|
+
command = "npx"
|
|
161
|
+
args = ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
**With Environment Variables:**
|
|
165
|
+
|
|
166
|
+
```toml
|
|
167
|
+
[mcp_servers.superfetch]
|
|
168
|
+
command = "npx"
|
|
169
|
+
args = ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
170
|
+
env = { CACHE_TTL = "7200", LOG_LEVEL = "debug", FETCH_TIMEOUT = "60000" }
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
> **Access config file:** Click the gear icon → "Codex Settings > Open config.toml"
|
|
174
|
+
>
|
|
175
|
+
> **Documentation:** [Codex MCP Guide](https://codex.com/docs/mcp)
|
|
176
|
+
|
|
177
|
+
</details>
|
|
178
|
+
|
|
148
179
|
<details>
|
|
149
180
|
<summary><strong>Cline (VS Code Extension)</strong></summary>
|
|
150
181
|
|
|
@@ -456,25 +487,94 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
|
|
|
456
487
|
|
|
457
488
|
## Configuration
|
|
458
489
|
|
|
459
|
-
###
|
|
490
|
+
### Configuration Presets
|
|
491
|
+
|
|
492
|
+
<details open>
|
|
493
|
+
<summary><strong>Default (Recommended)</strong> — No configuration needed</summary>
|
|
494
|
+
|
|
495
|
+
```json
|
|
496
|
+
{
|
|
497
|
+
"servers": {
|
|
498
|
+
"superFetch": {
|
|
499
|
+
"command": "npx",
|
|
500
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
```
|
|
505
|
+
|
|
506
|
+
</details>
|
|
460
507
|
|
|
461
508
|
<details>
|
|
462
|
-
<summary><strong>
|
|
509
|
+
<summary><strong>Debug Mode</strong> — Verbose logging and no cache</summary>
|
|
463
510
|
|
|
464
|
-
|
|
511
|
+
**VS Code** (`.vscode/mcp.json`):
|
|
465
512
|
|
|
466
|
-
```
|
|
467
|
-
|
|
513
|
+
```json
|
|
514
|
+
{
|
|
515
|
+
"servers": {
|
|
516
|
+
"superFetch": {
|
|
517
|
+
"command": "npx",
|
|
518
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
519
|
+
"env": {
|
|
520
|
+
"LOG_LEVEL": "debug",
|
|
521
|
+
"CACHE_ENABLED": "false"
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
**Claude Desktop** (`claude_desktop_config.json`):
|
|
529
|
+
|
|
530
|
+
```json
|
|
531
|
+
{
|
|
532
|
+
"mcpServers": {
|
|
533
|
+
"superFetch": {
|
|
534
|
+
"command": "npx",
|
|
535
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
536
|
+
"env": {
|
|
537
|
+
"LOG_LEVEL": "debug",
|
|
538
|
+
"CACHE_ENABLED": "false"
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
468
543
|
```
|
|
469
544
|
|
|
470
|
-
|
|
545
|
+
**Cursor** (MCP settings):
|
|
546
|
+
|
|
547
|
+
```json
|
|
548
|
+
{
|
|
549
|
+
"mcpServers": {
|
|
550
|
+
"superFetch": {
|
|
551
|
+
"command": "npx",
|
|
552
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
553
|
+
"env": {
|
|
554
|
+
"LOG_LEVEL": "debug",
|
|
555
|
+
"CACHE_ENABLED": "false"
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
```
|
|
561
|
+
|
|
562
|
+
</details>
|
|
563
|
+
|
|
564
|
+
<details>
|
|
565
|
+
<summary><strong>Performance Mode</strong> — Aggressive caching for speed</summary>
|
|
471
566
|
|
|
472
567
|
```json
|
|
473
568
|
{
|
|
474
569
|
"servers": {
|
|
475
570
|
"superFetch": {
|
|
476
|
-
"
|
|
477
|
-
"
|
|
571
|
+
"command": "npx",
|
|
572
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
573
|
+
"env": {
|
|
574
|
+
"CACHE_TTL": "7200",
|
|
575
|
+
"CACHE_MAX_KEYS": "500",
|
|
576
|
+
"LOG_LEVEL": "warn"
|
|
577
|
+
}
|
|
478
578
|
}
|
|
479
579
|
}
|
|
480
580
|
}
|
|
@@ -483,14 +583,38 @@ Then add to `.vscode/mcp.json`:
|
|
|
483
583
|
</details>
|
|
484
584
|
|
|
485
585
|
<details>
|
|
486
|
-
<summary><strong>
|
|
586
|
+
<summary><strong>Custom User Agent</strong> — For sites that block bots</summary>
|
|
487
587
|
|
|
488
588
|
```json
|
|
489
589
|
{
|
|
490
|
-
"
|
|
590
|
+
"servers": {
|
|
591
|
+
"superFetch": {
|
|
592
|
+
"command": "npx",
|
|
593
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
594
|
+
"env": {
|
|
595
|
+
"USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
```
|
|
601
|
+
|
|
602
|
+
</details>
|
|
603
|
+
|
|
604
|
+
<details>
|
|
605
|
+
<summary><strong>Slow Networks / CI/CD</strong> — Extended timeouts</summary>
|
|
606
|
+
|
|
607
|
+
```json
|
|
608
|
+
{
|
|
609
|
+
"servers": {
|
|
491
610
|
"superFetch": {
|
|
492
|
-
"command": "
|
|
493
|
-
"args": ["
|
|
611
|
+
"command": "npx",
|
|
612
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
613
|
+
"env": {
|
|
614
|
+
"FETCH_TIMEOUT": "60000",
|
|
615
|
+
"CACHE_ENABLED": "false",
|
|
616
|
+
"LOG_LEVEL": "warn"
|
|
617
|
+
}
|
|
494
618
|
}
|
|
495
619
|
}
|
|
496
620
|
}
|
|
@@ -498,25 +622,93 @@ Then add to `.vscode/mcp.json`:
|
|
|
498
622
|
|
|
499
623
|
</details>
|
|
500
624
|
|
|
501
|
-
### Environment Variables
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
|
508
|
-
|
|
|
509
|
-
| `
|
|
510
|
-
| `
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
|
515
|
-
|
|
|
516
|
-
| `
|
|
517
|
-
| `
|
|
518
|
-
| `
|
|
519
|
-
|
|
625
|
+
### Available Environment Variables
|
|
626
|
+
|
|
627
|
+
Configure SuperFetch behavior by adding environment variables to your MCP client configuration's `env` property.
|
|
628
|
+
|
|
629
|
+
#### 🌐 Fetcher Settings
|
|
630
|
+
|
|
631
|
+
| Variable | Default | Valid Values | Description |
|
|
632
|
+
| --------------- | -------------------- | -------------------- | --------------------------------------------------------------- |
|
|
633
|
+
| `FETCH_TIMEOUT` | `30000` | `5000`-`120000` | Request timeout in milliseconds (5s-2min) |
|
|
634
|
+
| `USER_AGENT` | `superFetch-MCP/1.0` | Any valid user agent | Custom user agent for requests (useful for sites blocking bots) |
|
|
635
|
+
|
|
636
|
+
#### 💾 Cache Settings
|
|
637
|
+
|
|
638
|
+
| Variable | Default | Valid Values | Description |
|
|
639
|
+
| ---------------- | ------- | ---------------- | -------------------------------------- |
|
|
640
|
+
| `CACHE_ENABLED` | `true` | `true` / `false` | Enable response caching |
|
|
641
|
+
| `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
|
|
642
|
+
| `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
|
|
643
|
+
|
|
644
|
+
#### 📝 Logging Settings
|
|
645
|
+
|
|
646
|
+
| Variable | Default | Valid Values | Description |
|
|
647
|
+
| ---------------- | ------- | ----------------------------------- | -------------------------- |
|
|
648
|
+
| `LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error` | Logging verbosity level |
|
|
649
|
+
| `ENABLE_LOGGING` | `true` | `true` / `false` | Enable/disable all logging |
|
|
650
|
+
|
|
651
|
+
#### 🔍 Extraction Settings
|
|
652
|
+
|
|
653
|
+
| Variable | Default | Valid Values | Description |
|
|
654
|
+
| ---------------------- | ------- | ---------------- | -------------------------------------------------- |
|
|
655
|
+
| `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
|
|
656
|
+
| `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
|
|
657
|
+
|
|
658
|
+
### HTTP Mode Configuration
|
|
659
|
+
|
|
660
|
+
<details>
|
|
661
|
+
<summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
|
|
662
|
+
|
|
663
|
+
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
|
|
664
|
+
|
|
665
|
+
#### Start HTTP Server
|
|
666
|
+
|
|
667
|
+
```bash
|
|
668
|
+
npx -y @j0hanz/superfetch@latest
|
|
669
|
+
# Server runs at http://127.0.0.1:3000
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
#### HTTP-Specific Environment Variables
|
|
673
|
+
|
|
674
|
+
| Variable | Default | Description |
|
|
675
|
+
| ----------------- | ----------- | ------------------------------------------------ |
|
|
676
|
+
| `PORT` | `3000` | HTTP server port |
|
|
677
|
+
| `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
|
|
678
|
+
| `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
|
|
679
|
+
| `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
|
|
680
|
+
|
|
681
|
+
#### VS Code HTTP Mode Setup
|
|
682
|
+
|
|
683
|
+
```json
|
|
684
|
+
{
|
|
685
|
+
"servers": {
|
|
686
|
+
"superFetch": {
|
|
687
|
+
"type": "http",
|
|
688
|
+
"url": "http://127.0.0.1:3000/mcp"
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
```
|
|
693
|
+
|
|
694
|
+
#### Docker/Kubernetes Example
|
|
695
|
+
|
|
696
|
+
```bash
|
|
697
|
+
PORT=8080 HOST=0.0.0.0 ALLOWED_ORIGINS=https://myapp.com npx @j0hanz/superfetch@latest
|
|
698
|
+
```
|
|
699
|
+
|
|
700
|
+
</details>
|
|
701
|
+
|
|
702
|
+
### Configuration Cookbook
|
|
703
|
+
|
|
704
|
+
| Use Case | Configuration |
|
|
705
|
+
| ---------------------------- | -------------------------------------------------------------- |
|
|
706
|
+
| 🐛 **Debugging issues** | `LOG_LEVEL=debug`, `CACHE_ENABLED=false` |
|
|
707
|
+
| 🚀 **Maximum performance** | `CACHE_TTL=7200`, `CACHE_MAX_KEYS=500`, `LOG_LEVEL=error` |
|
|
708
|
+
| 🌐 **Slow target sites** | `FETCH_TIMEOUT=60000` |
|
|
709
|
+
| 🤖 **Bypass bot detection** | `USER_AGENT="Mozilla/5.0 (compatible; MyBot/1.0)"` |
|
|
710
|
+
| 🔄 **CI/CD (always fresh)** | `CACHE_ENABLED=false`, `FETCH_TIMEOUT=60000`, `LOG_LEVEL=warn` |
|
|
711
|
+
| 📊 **Production monitoring** | `LOG_LEVEL=warn` or `error` |
|
|
520
712
|
|
|
521
713
|
---
|
|
522
714
|
|
|
@@ -592,7 +784,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
592
784
|
| ------------------ | --------------------------------- |
|
|
593
785
|
| Runtime | Node.js ≥20.0.0 |
|
|
594
786
|
| Language | TypeScript 5.9 |
|
|
595
|
-
| MCP SDK | @modelcontextprotocol/sdk ^1.
|
|
787
|
+
| MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
|
|
596
788
|
| Content Extraction | @mozilla/readability ^0.6.0 |
|
|
597
789
|
| HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
|
|
598
790
|
| Markdown | Turndown ^7.2.2 |
|
package/dist/config/index.d.ts
CHANGED
|
@@ -7,9 +7,9 @@ export declare const config: {
|
|
|
7
7
|
};
|
|
8
8
|
readonly fetcher: {
|
|
9
9
|
readonly timeout: number;
|
|
10
|
-
readonly maxRedirects:
|
|
10
|
+
readonly maxRedirects: 5;
|
|
11
11
|
readonly userAgent: string;
|
|
12
|
-
readonly maxContentLength:
|
|
12
|
+
readonly maxContentLength: 10485760;
|
|
13
13
|
};
|
|
14
14
|
readonly cache: {
|
|
15
15
|
readonly enabled: boolean;
|
|
@@ -19,11 +19,11 @@ export declare const config: {
|
|
|
19
19
|
readonly extraction: {
|
|
20
20
|
readonly extractMainContent: boolean;
|
|
21
21
|
readonly includeMetadata: boolean;
|
|
22
|
-
readonly maxBlockLength:
|
|
23
|
-
readonly minParagraphLength:
|
|
22
|
+
readonly maxBlockLength: 5000;
|
|
23
|
+
readonly minParagraphLength: 10;
|
|
24
24
|
};
|
|
25
25
|
readonly logging: {
|
|
26
|
-
readonly level:
|
|
26
|
+
readonly level: "debug" | "info" | "warn" | "error";
|
|
27
27
|
readonly enabled: boolean;
|
|
28
28
|
};
|
|
29
29
|
readonly constants: {
|
|
@@ -31,5 +31,10 @@ export declare const config: {
|
|
|
31
31
|
readonly maxContentSize: number;
|
|
32
32
|
readonly maxUrlLength: 2048;
|
|
33
33
|
};
|
|
34
|
+
readonly security: {
|
|
35
|
+
readonly blockedHosts: Set<string>;
|
|
36
|
+
readonly blockedIpPatterns: readonly RegExp[];
|
|
37
|
+
readonly blockedHeaders: Set<string>;
|
|
38
|
+
};
|
|
34
39
|
};
|
|
35
40
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAqCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCA0DV,SAAS,MAAM,EAAE;;;CAUhB,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,43 +1,97 @@
|
|
|
1
|
-
function
|
|
2
|
-
if (!
|
|
1
|
+
function parseInteger(envValue, defaultValue, min, max) {
|
|
2
|
+
if (!envValue)
|
|
3
3
|
return defaultValue;
|
|
4
|
-
const parsed = parseInt(
|
|
4
|
+
const parsed = parseInt(envValue, 10);
|
|
5
5
|
if (isNaN(parsed))
|
|
6
6
|
return defaultValue;
|
|
7
|
-
|
|
7
|
+
if (min !== undefined && parsed < min)
|
|
8
|
+
return defaultValue;
|
|
9
|
+
if (max !== undefined && parsed > max)
|
|
10
|
+
return defaultValue;
|
|
11
|
+
return parsed;
|
|
12
|
+
}
|
|
13
|
+
function parseBoolean(envValue, defaultValue) {
|
|
14
|
+
if (!envValue)
|
|
15
|
+
return defaultValue;
|
|
16
|
+
return envValue !== 'false';
|
|
17
|
+
}
|
|
18
|
+
function parseLogLevel(envValue) {
|
|
19
|
+
const level = envValue?.toLowerCase();
|
|
20
|
+
if (level === 'debug' ||
|
|
21
|
+
level === 'info' ||
|
|
22
|
+
level === 'warn' ||
|
|
23
|
+
level === 'error') {
|
|
24
|
+
return level;
|
|
25
|
+
}
|
|
26
|
+
return 'info';
|
|
8
27
|
}
|
|
9
28
|
export const config = {
|
|
10
29
|
server: {
|
|
11
30
|
name: 'superFetch',
|
|
12
31
|
version: '1.0.0',
|
|
13
|
-
port:
|
|
32
|
+
port: parseInteger(process.env.PORT, 3000, 1024, 65535),
|
|
14
33
|
host: process.env.HOST ?? '127.0.0.1',
|
|
15
34
|
},
|
|
16
35
|
fetcher: {
|
|
17
|
-
timeout:
|
|
18
|
-
maxRedirects:
|
|
36
|
+
timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
|
|
37
|
+
maxRedirects: 5,
|
|
19
38
|
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
|
|
20
|
-
maxContentLength:
|
|
39
|
+
maxContentLength: 10485760,
|
|
21
40
|
},
|
|
22
41
|
cache: {
|
|
23
|
-
enabled: process.env.CACHE_ENABLED
|
|
24
|
-
ttl:
|
|
25
|
-
maxKeys:
|
|
42
|
+
enabled: parseBoolean(process.env.CACHE_ENABLED, true),
|
|
43
|
+
ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
44
|
+
maxKeys: parseInteger(process.env.CACHE_MAX_KEYS, 100, 10, 1000),
|
|
26
45
|
},
|
|
27
46
|
extraction: {
|
|
28
|
-
extractMainContent: process.env.EXTRACT_MAIN_CONTENT
|
|
29
|
-
includeMetadata: process.env.INCLUDE_METADATA
|
|
30
|
-
maxBlockLength:
|
|
31
|
-
minParagraphLength:
|
|
47
|
+
extractMainContent: parseBoolean(process.env.EXTRACT_MAIN_CONTENT, true),
|
|
48
|
+
includeMetadata: parseBoolean(process.env.INCLUDE_METADATA, true),
|
|
49
|
+
maxBlockLength: 5000,
|
|
50
|
+
minParagraphLength: 10,
|
|
32
51
|
},
|
|
33
52
|
logging: {
|
|
34
|
-
level: process.env.LOG_LEVEL
|
|
35
|
-
enabled: process.env.ENABLE_LOGGING
|
|
53
|
+
level: parseLogLevel(process.env.LOG_LEVEL),
|
|
54
|
+
enabled: parseBoolean(process.env.ENABLE_LOGGING, true),
|
|
36
55
|
},
|
|
37
56
|
constants: {
|
|
38
|
-
maxHtmlSize: 10 * 1024 * 1024,
|
|
39
|
-
maxContentSize: 5 * 1024 * 1024,
|
|
57
|
+
maxHtmlSize: 10 * 1024 * 1024,
|
|
58
|
+
maxContentSize: 5 * 1024 * 1024,
|
|
40
59
|
maxUrlLength: 2048,
|
|
41
60
|
},
|
|
61
|
+
security: {
|
|
62
|
+
blockedHosts: new Set([
|
|
63
|
+
'localhost',
|
|
64
|
+
'127.0.0.1',
|
|
65
|
+
'0.0.0.0',
|
|
66
|
+
'::1',
|
|
67
|
+
'169.254.169.254',
|
|
68
|
+
'metadata.google.internal',
|
|
69
|
+
'metadata.azure.com',
|
|
70
|
+
'100.100.100.200',
|
|
71
|
+
'instance-data',
|
|
72
|
+
]),
|
|
73
|
+
blockedIpPatterns: [
|
|
74
|
+
/^10\./,
|
|
75
|
+
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
76
|
+
/^192\.168\./,
|
|
77
|
+
/^127\./,
|
|
78
|
+
/^0\./,
|
|
79
|
+
/^169\.254\./,
|
|
80
|
+
/^fc00:/i,
|
|
81
|
+
/^fe80:/i,
|
|
82
|
+
/^::ffff:127\./,
|
|
83
|
+
/^::ffff:10\./,
|
|
84
|
+
/^::ffff:172\.(1[6-9]|2\d|3[01])\./,
|
|
85
|
+
/^::ffff:192\.168\./,
|
|
86
|
+
],
|
|
87
|
+
blockedHeaders: new Set([
|
|
88
|
+
'host',
|
|
89
|
+
'authorization',
|
|
90
|
+
'cookie',
|
|
91
|
+
'x-forwarded-for',
|
|
92
|
+
'x-real-ip',
|
|
93
|
+
'proxy-authorization',
|
|
94
|
+
]),
|
|
95
|
+
},
|
|
42
96
|
};
|
|
43
97
|
//# sourceMappingURL=index.js.map
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;KACH;CACO,CAAC"}
|