@j0hanz/superfetch 1.0.6 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +228 -36
  2. package/dist/config/index.d.ts +10 -5
  3. package/dist/config/index.d.ts.map +1 -1
  4. package/dist/config/index.js +73 -19
  5. package/dist/config/index.js.map +1 -1
  6. package/dist/config/types.d.ts +98 -57
  7. package/dist/config/types.d.ts.map +1 -1
  8. package/dist/errors/app-error.d.ts +4 -28
  9. package/dist/errors/app-error.d.ts.map +1 -1
  10. package/dist/errors/app-error.js +10 -51
  11. package/dist/errors/app-error.js.map +1 -1
  12. package/dist/index.js +10 -55
  13. package/dist/index.js.map +1 -1
  14. package/dist/middleware/error-handler.d.ts +2 -2
  15. package/dist/middleware/error-handler.d.ts.map +1 -1
  16. package/dist/middleware/error-handler.js +12 -14
  17. package/dist/middleware/error-handler.js.map +1 -1
  18. package/dist/middleware/rate-limiter.d.ts.map +1 -1
  19. package/dist/middleware/rate-limiter.js +0 -8
  20. package/dist/middleware/rate-limiter.js.map +1 -1
  21. package/dist/parsers/base-html-element-parser.d.ts +43 -0
  22. package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
  23. package/dist/parsers/base-html-element-parser.js +59 -0
  24. package/dist/parsers/base-html-element-parser.js.map +1 -0
  25. package/dist/parsers/heading-element-parser.d.ts +14 -0
  26. package/dist/parsers/heading-element-parser.d.ts.map +1 -0
  27. package/dist/parsers/heading-element-parser.js +26 -0
  28. package/dist/parsers/heading-element-parser.js.map +1 -0
  29. package/dist/parsers/image-element-parser.d.ts +16 -0
  30. package/dist/parsers/image-element-parser.d.ts.map +1 -0
  31. package/dist/parsers/image-element-parser.js +33 -0
  32. package/dist/parsers/image-element-parser.js.map +1 -0
  33. package/dist/parsers/link-element-parser.d.ts +15 -0
  34. package/dist/parsers/link-element-parser.d.ts.map +1 -0
  35. package/dist/parsers/link-element-parser.js +28 -0
  36. package/dist/parsers/link-element-parser.js.map +1 -0
  37. package/dist/parsers/open-graph-parser.d.ts +17 -0
  38. package/dist/parsers/open-graph-parser.d.ts.map +1 -0
  39. package/dist/parsers/open-graph-parser.js +41 -0
  40. package/dist/parsers/open-graph-parser.js.map +1 -0
  41. package/dist/parsers/schema-org-parser.d.ts +17 -0
  42. package/dist/parsers/schema-org-parser.d.ts.map +1 -0
  43. package/dist/parsers/schema-org-parser.js +32 -0
  44. package/dist/parsers/schema-org-parser.js.map +1 -0
  45. package/dist/parsers/standard-meta-parser.d.ts +18 -0
  46. package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
  47. package/dist/parsers/standard-meta-parser.js +32 -0
  48. package/dist/parsers/standard-meta-parser.js.map +1 -0
  49. package/dist/parsers/twitter-card-parser.d.ts +17 -0
  50. package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
  51. package/dist/parsers/twitter-card-parser.js +41 -0
  52. package/dist/parsers/twitter-card-parser.js.map +1 -0
  53. package/dist/resources/cached-content.d.ts +0 -1
  54. package/dist/resources/cached-content.d.ts.map +1 -1
  55. package/dist/resources/cached-content.js +3 -9
  56. package/dist/resources/cached-content.js.map +1 -1
  57. package/dist/resources/index.d.ts.map +1 -1
  58. package/dist/resources/index.js +8 -8
  59. package/dist/resources/index.js.map +1 -1
  60. package/dist/server.d.ts.map +1 -1
  61. package/dist/server.js +10 -10
  62. package/dist/server.js.map +1 -1
  63. package/dist/services/cache.d.ts +0 -28
  64. package/dist/services/cache.d.ts.map +1 -1
  65. package/dist/services/cache.js +10 -173
  66. package/dist/services/cache.js.map +1 -1
  67. package/dist/services/extractor.d.ts +1 -11
  68. package/dist/services/extractor.d.ts.map +1 -1
  69. package/dist/services/extractor.js +86 -84
  70. package/dist/services/extractor.js.map +1 -1
  71. package/dist/services/fetcher.d.ts +2 -13
  72. package/dist/services/fetcher.d.ts.map +1 -1
  73. package/dist/services/fetcher.js +195 -211
  74. package/dist/services/fetcher.js.map +1 -1
  75. package/dist/services/logger.d.ts +5 -4
  76. package/dist/services/logger.d.ts.map +1 -1
  77. package/dist/services/logger.js +27 -42
  78. package/dist/services/logger.js.map +1 -1
  79. package/dist/services/parser.d.ts.map +1 -1
  80. package/dist/services/parser.js +35 -26
  81. package/dist/services/parser.js.map +1 -1
  82. package/dist/services/session-manager.d.ts +18 -0
  83. package/dist/services/session-manager.d.ts.map +1 -0
  84. package/dist/services/session-manager.js +73 -0
  85. package/dist/services/session-manager.js.map +1 -0
  86. package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
  87. package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
  88. package/dist/strategies/exponential-backoff-strategy.js +32 -0
  89. package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
  90. package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
  91. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  92. package/dist/tools/handlers/fetch-links.tool.js +0 -1
  93. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  94. package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
  95. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  96. package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
  97. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  98. package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
  99. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  100. package/dist/tools/handlers/fetch-url.tool.js +15 -20
  101. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  102. package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
  103. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  104. package/dist/tools/handlers/fetch-urls.tool.js +124 -105
  105. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  106. package/dist/tools/index.d.ts.map +1 -1
  107. package/dist/tools/index.js +0 -4
  108. package/dist/tools/index.js.map +1 -1
  109. package/dist/tools/utils/common.d.ts +6 -7
  110. package/dist/tools/utils/common.d.ts.map +1 -1
  111. package/dist/tools/utils/common.js +8 -8
  112. package/dist/tools/utils/common.js.map +1 -1
  113. package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
  114. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  115. package/dist/tools/utils/fetch-pipeline.js +47 -79
  116. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  117. package/dist/transformers/jsonl.transformer.d.ts +1 -1
  118. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  119. package/dist/transformers/jsonl.transformer.js +15 -10
  120. package/dist/transformers/jsonl.transformer.js.map +1 -1
  121. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  122. package/dist/transformers/markdown.transformer.js +58 -62
  123. package/dist/transformers/markdown.transformer.js.map +1 -1
  124. package/dist/utils/concurrency.d.ts +2 -5
  125. package/dist/utils/concurrency.d.ts.map +1 -1
  126. package/dist/utils/concurrency.js +19 -19
  127. package/dist/utils/concurrency.js.map +1 -1
  128. package/dist/utils/content-cleaner.d.ts +0 -25
  129. package/dist/utils/content-cleaner.d.ts.map +1 -1
  130. package/dist/utils/content-cleaner.js +12 -187
  131. package/dist/utils/content-cleaner.js.map +1 -1
  132. package/dist/utils/html-truncator.d.ts +2 -0
  133. package/dist/utils/html-truncator.d.ts.map +1 -0
  134. package/dist/utils/html-truncator.js +14 -0
  135. package/dist/utils/html-truncator.js.map +1 -0
  136. package/dist/utils/language-detector.d.ts +0 -3
  137. package/dist/utils/language-detector.d.ts.map +1 -1
  138. package/dist/utils/language-detector.js +0 -11
  139. package/dist/utils/language-detector.js.map +1 -1
  140. package/dist/utils/sanitizer.d.ts.map +1 -1
  141. package/dist/utils/sanitizer.js +7 -5
  142. package/dist/utils/sanitizer.js.map +1 -1
  143. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  144. package/dist/utils/tool-error-handler.js +15 -42
  145. package/dist/utils/tool-error-handler.js.map +1 -1
  146. package/dist/utils/url-validator.d.ts +0 -6
  147. package/dist/utils/url-validator.d.ts.map +1 -1
  148. package/dist/utils/url-validator.js +12 -81
  149. package/dist/utils/url-validator.js.map +1 -1
  150. package/package.json +5 -6
package/README.md CHANGED
@@ -12,8 +12,6 @@
12
12
 
13
13
  A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
14
14
 
15
- **Version:** 1.0.5
16
-
17
15
  [Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
18
16
 
19
17
  > 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
@@ -108,7 +106,9 @@ Add to `.vscode/mcp.json` in your workspace:
108
106
  }
109
107
  ```
110
108
 
111
- ### With Environment Variables
109
+ ### With Custom Configuration
110
+
111
+ Configure SuperFetch behavior by adding environment variables to the `env` property:
112
112
 
113
113
  ```json
114
114
  {
@@ -118,13 +118,16 @@ Add to `.vscode/mcp.json` in your workspace:
118
118
  "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
119
119
  "env": {
120
120
  "CACHE_TTL": "7200",
121
- "LOG_LEVEL": "debug"
121
+ "LOG_LEVEL": "debug",
122
+ "FETCH_TIMEOUT": "60000"
122
123
  }
123
124
  }
124
125
  }
125
126
  }
126
127
  ```
127
128
 
129
+ See [Configuration](#configuration) section below for all available options and presets.
130
+
128
131
  ### Cursor
129
132
 
130
133
  1. Open Cursor Settings
@@ -145,6 +148,34 @@ Add to `.vscode/mcp.json` in your workspace:
145
148
 
146
149
  > **Tip:** On Windows, if you encounter issues, try: `cmd /c "npx -y @j0hanz/superfetch@latest --stdio"`
147
150
 
151
+ <details>
152
+ <summary><strong>Codex IDE</strong></summary>
153
+
154
+ Add to your `~/.codex/config.toml` file:
155
+
156
+ **Basic Configuration:**
157
+
158
+ ```toml
159
+ [mcp_servers.superfetch]
160
+ command = "npx"
161
+ args = ["-y", "@j0hanz/superfetch@latest", "--stdio"]
162
+ ```
163
+
164
+ **With Environment Variables:**
165
+
166
+ ```toml
167
+ [mcp_servers.superfetch]
168
+ command = "npx"
169
+ args = ["-y", "@j0hanz/superfetch@latest", "--stdio"]
170
+ env = { CACHE_TTL = "7200", LOG_LEVEL = "debug", FETCH_TIMEOUT = "60000" }
171
+ ```
172
+
173
+ > **Access config file:** Click the gear icon → "Codex Settings &gt; Open config.toml"
174
+ >
175
+ > **Documentation:** [Codex MCP Guide](https://codex.com/docs/mcp)
176
+
177
+ </details>
178
+
148
179
  <details>
149
180
  <summary><strong>Cline (VS Code Extension)</strong></summary>
150
181
 
@@ -456,25 +487,94 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
456
487
 
457
488
  ## Configuration
458
489
 
459
- ### Alternative MCP Client Setups
490
+ ### Configuration Presets
491
+
492
+ <details open>
493
+ <summary><strong>Default (Recommended)</strong> — No configuration needed</summary>
494
+
495
+ ```json
496
+ {
497
+ "servers": {
498
+ "superFetch": {
499
+ "command": "npx",
500
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
501
+ }
502
+ }
503
+ }
504
+ ```
505
+
506
+ </details>
460
507
 
461
508
  <details>
462
- <summary><strong>VS Code (HTTP mode)</strong> — requires running server separately</summary>
509
+ <summary><strong>Debug Mode</strong> — Verbose logging and no cache</summary>
463
510
 
464
- First, start the HTTP server:
511
+ **VS Code** (`.vscode/mcp.json`):
465
512
 
466
- ```bash
467
- npx -y @j0hanz/superfetch@latest
513
+ ```json
514
+ {
515
+ "servers": {
516
+ "superFetch": {
517
+ "command": "npx",
518
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
519
+ "env": {
520
+ "LOG_LEVEL": "debug",
521
+ "CACHE_ENABLED": "false"
522
+ }
523
+ }
524
+ }
525
+ }
526
+ ```
527
+
528
+ **Claude Desktop** (`claude_desktop_config.json`):
529
+
530
+ ```json
531
+ {
532
+ "mcpServers": {
533
+ "superFetch": {
534
+ "command": "npx",
535
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
536
+ "env": {
537
+ "LOG_LEVEL": "debug",
538
+ "CACHE_ENABLED": "false"
539
+ }
540
+ }
541
+ }
542
+ }
468
543
  ```
469
544
 
470
- Then add to `.vscode/mcp.json`:
545
+ **Cursor** (MCP settings):
546
+
547
+ ```json
548
+ {
549
+ "mcpServers": {
550
+ "superFetch": {
551
+ "command": "npx",
552
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
553
+ "env": {
554
+ "LOG_LEVEL": "debug",
555
+ "CACHE_ENABLED": "false"
556
+ }
557
+ }
558
+ }
559
+ }
560
+ ```
561
+
562
+ </details>
563
+
564
+ <details>
565
+ <summary><strong>Performance Mode</strong> — Aggressive caching for speed</summary>
471
566
 
472
567
  ```json
473
568
  {
474
569
  "servers": {
475
570
  "superFetch": {
476
- "type": "http",
477
- "url": "http://127.0.0.1:3000/mcp"
571
+ "command": "npx",
572
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
573
+ "env": {
574
+ "CACHE_TTL": "7200",
575
+ "CACHE_MAX_KEYS": "500",
576
+ "LOG_LEVEL": "warn"
577
+ }
478
578
  }
479
579
  }
480
580
  }
@@ -483,14 +583,38 @@ Then add to `.vscode/mcp.json`:
483
583
  </details>
484
584
 
485
585
  <details>
486
- <summary><strong>Claude Desktop (local path)</strong> — for development</summary>
586
+ <summary><strong>Custom User Agent</strong> — For sites that block bots</summary>
487
587
 
488
588
  ```json
489
589
  {
490
- "mcpServers": {
590
+ "servers": {
591
+ "superFetch": {
592
+ "command": "npx",
593
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
594
+ "env": {
595
+ "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
596
+ }
597
+ }
598
+ }
599
+ }
600
+ ```
601
+
602
+ </details>
603
+
604
+ <details>
605
+ <summary><strong>Slow Networks / CI/CD</strong> — Extended timeouts</summary>
606
+
607
+ ```json
608
+ {
609
+ "servers": {
491
610
  "superFetch": {
492
- "command": "node",
493
- "args": ["/path/to/super-fetch-mcp-server/dist/index.js", "--stdio"]
611
+ "command": "npx",
612
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
613
+ "env": {
614
+ "FETCH_TIMEOUT": "60000",
615
+ "CACHE_ENABLED": "false",
616
+ "LOG_LEVEL": "warn"
617
+ }
494
618
  }
495
619
  }
496
620
  }
@@ -498,25 +622,93 @@ Then add to `.vscode/mcp.json`:
498
622
 
499
623
  </details>
500
624
 
501
- ### Environment Variables
502
-
503
- | Variable | Default | Description |
504
- | ---------------------- | -------------------- | ------------------------------- |
505
- | `PORT` | `3000` | HTTP server port |
506
- | `HOST` | `127.0.0.1` | HTTP server host |
507
- | `FETCH_TIMEOUT` | `30000` | Request timeout (ms) |
508
- | `MAX_REDIRECTS` | `5` | Maximum HTTP redirects |
509
- | `USER_AGENT` | `superFetch-MCP/1.0` | HTTP User-Agent |
510
- | `MAX_CONTENT_LENGTH` | `10485760` | Max response size (bytes) |
511
- | `CACHE_ENABLED` | `true` | Enable response caching |
512
- | `CACHE_TTL` | `3600` | Cache TTL (seconds) |
513
- | `CACHE_MAX_KEYS` | `100` | Maximum cache entries |
514
- | `LOG_LEVEL` | `info` | Logging level |
515
- | `ENABLE_LOGGING` | `true` | Enable/disable logging |
516
- | `EXTRACT_MAIN_CONTENT` | `true` | Extract main content by default |
517
- | `INCLUDE_METADATA` | `true` | Include metadata by default |
518
- | `MAX_BLOCK_LENGTH` | `5000` | Maximum block length |
519
- | `MIN_PARAGRAPH_LENGTH` | `10` | Minimum paragraph length |
625
+ ### Available Environment Variables
626
+
627
+ Configure SuperFetch behavior by adding environment variables to your MCP client configuration's `env` property.
628
+
629
+ #### 🌐 Fetcher Settings
630
+
631
+ | Variable | Default | Valid Values | Description |
632
+ | --------------- | -------------------- | -------------------- | --------------------------------------------------------------- |
633
+ | `FETCH_TIMEOUT` | `30000` | `5000`-`120000` | Request timeout in milliseconds (5s-2min) |
634
+ | `USER_AGENT` | `superFetch-MCP/1.0` | Any valid user agent | Custom user agent for requests (useful for sites blocking bots) |
635
+
636
+ #### 💾 Cache Settings
637
+
638
+ | Variable | Default | Valid Values | Description |
639
+ | ---------------- | ------- | ---------------- | -------------------------------------- |
640
+ | `CACHE_ENABLED` | `true` | `true` / `false` | Enable response caching |
641
+ | `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
642
+ | `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
643
+
644
+ #### 📝 Logging Settings
645
+
646
+ | Variable | Default | Valid Values | Description |
647
+ | ---------------- | ------- | ----------------------------------- | -------------------------- |
648
+ | `LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error` | Logging verbosity level |
649
+ | `ENABLE_LOGGING` | `true` | `true` / `false` | Enable/disable all logging |
650
+
651
+ #### 🔍 Extraction Settings
652
+
653
+ | Variable | Default | Valid Values | Description |
654
+ | ---------------------- | ------- | ---------------- | -------------------------------------------------- |
655
+ | `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
656
+ | `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
657
+
658
+ ### HTTP Mode Configuration
659
+
660
+ <details>
661
+ <summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
662
+
663
+ SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
664
+
665
+ #### Start HTTP Server
666
+
667
+ ```bash
668
+ npx -y @j0hanz/superfetch@latest
669
+ # Server runs at http://127.0.0.1:3000
670
+ ```
671
+
672
+ #### HTTP-Specific Environment Variables
673
+
674
+ | Variable | Default | Description |
675
+ | ----------------- | ----------- | ------------------------------------------------ |
676
+ | `PORT` | `3000` | HTTP server port |
677
+ | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
678
+ | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
679
+ | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
680
+
681
+ #### VS Code HTTP Mode Setup
682
+
683
+ ```json
684
+ {
685
+ "servers": {
686
+ "superFetch": {
687
+ "type": "http",
688
+ "url": "http://127.0.0.1:3000/mcp"
689
+ }
690
+ }
691
+ }
692
+ ```
693
+
694
+ #### Docker/Kubernetes Example
695
+
696
+ ```bash
697
+ PORT=8080 HOST=0.0.0.0 ALLOWED_ORIGINS=https://myapp.com npx @j0hanz/superfetch@latest
698
+ ```
699
+
700
+ </details>
701
+
702
+ ### Configuration Cookbook
703
+
704
+ | Use Case | Configuration |
705
+ | ---------------------------- | -------------------------------------------------------------- |
706
+ | 🐛 **Debugging issues** | `LOG_LEVEL=debug`, `CACHE_ENABLED=false` |
707
+ | 🚀 **Maximum performance** | `CACHE_TTL=7200`, `CACHE_MAX_KEYS=500`, `LOG_LEVEL=error` |
708
+ | 🌐 **Slow target sites** | `FETCH_TIMEOUT=60000` |
709
+ | 🤖 **Bypass bot detection** | `USER_AGENT="Mozilla/5.0 (compatible; MyBot/1.0)"` |
710
+ | 🔄 **CI/CD (always fresh)** | `CACHE_ENABLED=false`, `FETCH_TIMEOUT=60000`, `LOG_LEVEL=warn` |
711
+ | 📊 **Production monitoring** | `LOG_LEVEL=warn` or `error` |
520
712
 
521
713
  ---
522
714
 
@@ -592,7 +784,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
592
784
  | ------------------ | --------------------------------- |
593
785
  | Runtime | Node.js ≥20.0.0 |
594
786
  | Language | TypeScript 5.9 |
595
- | MCP SDK | @modelcontextprotocol/sdk ^1.24.3 |
787
+ | MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
596
788
  | Content Extraction | @mozilla/readability ^0.6.0 |
597
789
  | HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
598
790
  | Markdown | Turndown ^7.2.2 |
@@ -7,9 +7,9 @@ export declare const config: {
7
7
  };
8
8
  readonly fetcher: {
9
9
  readonly timeout: number;
10
- readonly maxRedirects: number;
10
+ readonly maxRedirects: 5;
11
11
  readonly userAgent: string;
12
- readonly maxContentLength: number;
12
+ readonly maxContentLength: 10485760;
13
13
  };
14
14
  readonly cache: {
15
15
  readonly enabled: boolean;
@@ -19,11 +19,11 @@ export declare const config: {
19
19
  readonly extraction: {
20
20
  readonly extractMainContent: boolean;
21
21
  readonly includeMetadata: boolean;
22
- readonly maxBlockLength: number;
23
- readonly minParagraphLength: number;
22
+ readonly maxBlockLength: 5000;
23
+ readonly minParagraphLength: 10;
24
24
  };
25
25
  readonly logging: {
26
- readonly level: string;
26
+ readonly level: "debug" | "info" | "warn" | "error";
27
27
  readonly enabled: boolean;
28
28
  };
29
29
  readonly constants: {
@@ -31,5 +31,10 @@ export declare const config: {
31
31
  readonly maxContentSize: number;
32
32
  readonly maxUrlLength: 2048;
33
33
  };
34
+ readonly security: {
35
+ readonly blockedHosts: Set<string>;
36
+ readonly blockedIpPatterns: readonly RegExp[];
37
+ readonly blockedHeaders: Set<string>;
38
+ };
34
39
  };
35
40
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CT,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAqCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oCA0DV,SAAS,MAAM,EAAE;;;CAUhB,CAAC"}
@@ -1,43 +1,97 @@
1
- function parseIntEnv(value, defaultValue, min = 0, max = Number.MAX_SAFE_INTEGER) {
2
- if (!value)
1
+ function parseInteger(envValue, defaultValue, min, max) {
2
+ if (!envValue)
3
3
  return defaultValue;
4
- const parsed = parseInt(value, 10);
4
+ const parsed = parseInt(envValue, 10);
5
5
  if (isNaN(parsed))
6
6
  return defaultValue;
7
- return Math.max(min, Math.min(max, parsed));
7
+ if (min !== undefined && parsed < min)
8
+ return defaultValue;
9
+ if (max !== undefined && parsed > max)
10
+ return defaultValue;
11
+ return parsed;
12
+ }
13
+ function parseBoolean(envValue, defaultValue) {
14
+ if (!envValue)
15
+ return defaultValue;
16
+ return envValue !== 'false';
17
+ }
18
+ function parseLogLevel(envValue) {
19
+ const level = envValue?.toLowerCase();
20
+ if (level === 'debug' ||
21
+ level === 'info' ||
22
+ level === 'warn' ||
23
+ level === 'error') {
24
+ return level;
25
+ }
26
+ return 'info';
8
27
  }
9
28
  export const config = {
10
29
  server: {
11
30
  name: 'superFetch',
12
31
  version: '1.0.0',
13
- port: parseIntEnv(process.env.PORT, 3000, 1, 65535),
32
+ port: parseInteger(process.env.PORT, 3000, 1024, 65535),
14
33
  host: process.env.HOST ?? '127.0.0.1',
15
34
  },
16
35
  fetcher: {
17
- timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
18
- maxRedirects: parseIntEnv(process.env.MAX_REDIRECTS, 5, 0, 20),
36
+ timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
37
+ maxRedirects: 5,
19
38
  userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
20
- maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
39
+ maxContentLength: 10485760,
21
40
  },
22
41
  cache: {
23
- enabled: process.env.CACHE_ENABLED !== 'false',
24
- ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
25
- maxKeys: parseIntEnv(process.env.CACHE_MAX_KEYS, 100, 10, 10000),
42
+ enabled: parseBoolean(process.env.CACHE_ENABLED, true),
43
+ ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
44
+ maxKeys: parseInteger(process.env.CACHE_MAX_KEYS, 100, 10, 1000),
26
45
  },
27
46
  extraction: {
28
- extractMainContent: process.env.EXTRACT_MAIN_CONTENT !== 'false',
29
- includeMetadata: process.env.INCLUDE_METADATA !== 'false',
30
- maxBlockLength: parseIntEnv(process.env.MAX_BLOCK_LENGTH, 5000, 100, 50000),
31
- minParagraphLength: parseIntEnv(process.env.MIN_PARAGRAPH_LENGTH, 10, 0, 1000),
47
+ extractMainContent: parseBoolean(process.env.EXTRACT_MAIN_CONTENT, true),
48
+ includeMetadata: parseBoolean(process.env.INCLUDE_METADATA, true),
49
+ maxBlockLength: 5000,
50
+ minParagraphLength: 10,
32
51
  },
33
52
  logging: {
34
- level: process.env.LOG_LEVEL ?? 'info',
35
- enabled: process.env.ENABLE_LOGGING !== 'false',
53
+ level: parseLogLevel(process.env.LOG_LEVEL),
54
+ enabled: parseBoolean(process.env.ENABLE_LOGGING, true),
36
55
  },
37
56
  constants: {
38
- maxHtmlSize: 10 * 1024 * 1024, // 10MB
39
- maxContentSize: 5 * 1024 * 1024, // 5MB
57
+ maxHtmlSize: 10 * 1024 * 1024,
58
+ maxContentSize: 5 * 1024 * 1024,
40
59
  maxUrlLength: 2048,
41
60
  },
61
+ security: {
62
+ blockedHosts: new Set([
63
+ 'localhost',
64
+ '127.0.0.1',
65
+ '0.0.0.0',
66
+ '::1',
67
+ '169.254.169.254',
68
+ 'metadata.google.internal',
69
+ 'metadata.azure.com',
70
+ '100.100.100.200',
71
+ 'instance-data',
72
+ ]),
73
+ blockedIpPatterns: [
74
+ /^10\./,
75
+ /^172\.(1[6-9]|2\d|3[01])\./,
76
+ /^192\.168\./,
77
+ /^127\./,
78
+ /^0\./,
79
+ /^169\.254\./,
80
+ /^fc00:/i,
81
+ /^fe80:/i,
82
+ /^::ffff:127\./,
83
+ /^::ffff:10\./,
84
+ /^::ffff:172\.(1[6-9]|2\d|3[01])\./,
85
+ /^::ffff:192\.168\./,
86
+ ],
87
+ blockedHeaders: new Set([
88
+ 'host',
89
+ 'authorization',
90
+ 'cookie',
91
+ 'x-forwarded-for',
92
+ 'x-real-ip',
93
+ 'proxy-authorization',
94
+ ]),
95
+ },
42
96
  };
43
97
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACpE,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT;KACF;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,OAAO;QACtC,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,MAAM;QACvC,YAAY,EAAE,IAAI;KACnB;CACO,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;IACD,QAAQ,EAAE;QACR,YAAY,EAAE,IAAI,GAAG,CAAC;YACpB,WAAW;YACX,WAAW;YACX,SAAS;YACT,KAAK;YACL,iBAAiB;YACjB,0BAA0B;YAC1B,oBAAoB;YACpB,iBAAiB;YACjB,eAAe;SAChB,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO;YACP,4BAA4B;YAC5B,aAAa;YACb,QAAQ;YACR,MAAM;YACN,aAAa;YACb,SAAS;YACT,SAAS;YACT,eAAe;YACf,cAAc;YACd,mCAAmC;YACnC,oBAAoB;SACA;QACtB,cAAc,EAAE,IAAI,GAAG,CAAC;YACtB,MAAM;YACN,eAAe;YACf,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,qBAAqB;SACtB,CAAC;KACH;CACO,CAAC"}