@j0hanz/superfetch 1.0.5 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/README.md +200 -36
  2. package/dist/config/index.d.ts +10 -5
  3. package/dist/config/index.d.ts.map +1 -1
  4. package/dist/config/index.js +41 -17
  5. package/dist/config/index.js.map +1 -1
  6. package/dist/config/types.d.ts +98 -57
  7. package/dist/config/types.d.ts.map +1 -1
  8. package/dist/errors/app-error.d.ts +4 -28
  9. package/dist/errors/app-error.d.ts.map +1 -1
  10. package/dist/errors/app-error.js +10 -51
  11. package/dist/errors/app-error.js.map +1 -1
  12. package/dist/index.js +31 -46
  13. package/dist/index.js.map +1 -1
  14. package/dist/middleware/error-handler.d.ts +2 -2
  15. package/dist/middleware/error-handler.d.ts.map +1 -1
  16. package/dist/middleware/error-handler.js +12 -14
  17. package/dist/middleware/error-handler.js.map +1 -1
  18. package/dist/middleware/rate-limiter.d.ts.map +1 -1
  19. package/dist/middleware/rate-limiter.js +31 -14
  20. package/dist/middleware/rate-limiter.js.map +1 -1
  21. package/dist/parsers/base-html-element-parser.d.ts +43 -0
  22. package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
  23. package/dist/parsers/base-html-element-parser.js +59 -0
  24. package/dist/parsers/base-html-element-parser.js.map +1 -0
  25. package/dist/parsers/heading-element-parser.d.ts +14 -0
  26. package/dist/parsers/heading-element-parser.d.ts.map +1 -0
  27. package/dist/parsers/heading-element-parser.js +26 -0
  28. package/dist/parsers/heading-element-parser.js.map +1 -0
  29. package/dist/parsers/image-element-parser.d.ts +16 -0
  30. package/dist/parsers/image-element-parser.d.ts.map +1 -0
  31. package/dist/parsers/image-element-parser.js +33 -0
  32. package/dist/parsers/image-element-parser.js.map +1 -0
  33. package/dist/parsers/link-element-parser.d.ts +15 -0
  34. package/dist/parsers/link-element-parser.d.ts.map +1 -0
  35. package/dist/parsers/link-element-parser.js +28 -0
  36. package/dist/parsers/link-element-parser.js.map +1 -0
  37. package/dist/parsers/open-graph-parser.d.ts +17 -0
  38. package/dist/parsers/open-graph-parser.d.ts.map +1 -0
  39. package/dist/parsers/open-graph-parser.js +41 -0
  40. package/dist/parsers/open-graph-parser.js.map +1 -0
  41. package/dist/parsers/schema-org-parser.d.ts +17 -0
  42. package/dist/parsers/schema-org-parser.d.ts.map +1 -0
  43. package/dist/parsers/schema-org-parser.js +32 -0
  44. package/dist/parsers/schema-org-parser.js.map +1 -0
  45. package/dist/parsers/standard-meta-parser.d.ts +18 -0
  46. package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
  47. package/dist/parsers/standard-meta-parser.js +32 -0
  48. package/dist/parsers/standard-meta-parser.js.map +1 -0
  49. package/dist/parsers/twitter-card-parser.d.ts +17 -0
  50. package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
  51. package/dist/parsers/twitter-card-parser.js +41 -0
  52. package/dist/parsers/twitter-card-parser.js.map +1 -0
  53. package/dist/resources/cached-content.d.ts +0 -2
  54. package/dist/resources/cached-content.d.ts.map +1 -1
  55. package/dist/resources/cached-content.js +3 -34
  56. package/dist/resources/cached-content.js.map +1 -1
  57. package/dist/resources/index.d.ts.map +1 -1
  58. package/dist/resources/index.js +8 -8
  59. package/dist/resources/index.js.map +1 -1
  60. package/dist/server.d.ts.map +1 -1
  61. package/dist/server.js +12 -11
  62. package/dist/server.js.map +1 -1
  63. package/dist/services/cache.d.ts +0 -28
  64. package/dist/services/cache.d.ts.map +1 -1
  65. package/dist/services/cache.js +10 -166
  66. package/dist/services/cache.js.map +1 -1
  67. package/dist/services/card-extractor.d.ts +0 -4
  68. package/dist/services/card-extractor.d.ts.map +1 -1
  69. package/dist/services/card-extractor.js +6 -1
  70. package/dist/services/card-extractor.js.map +1 -1
  71. package/dist/services/extractor.d.ts +1 -11
  72. package/dist/services/extractor.d.ts.map +1 -1
  73. package/dist/services/extractor.js +86 -84
  74. package/dist/services/extractor.js.map +1 -1
  75. package/dist/services/fetcher.d.ts +2 -13
  76. package/dist/services/fetcher.d.ts.map +1 -1
  77. package/dist/services/fetcher.js +79 -79
  78. package/dist/services/fetcher.js.map +1 -1
  79. package/dist/services/logger.d.ts +5 -4
  80. package/dist/services/logger.d.ts.map +1 -1
  81. package/dist/services/logger.js +27 -42
  82. package/dist/services/logger.js.map +1 -1
  83. package/dist/services/parser.d.ts.map +1 -1
  84. package/dist/services/parser.js +35 -27
  85. package/dist/services/parser.js.map +1 -1
  86. package/dist/services/session-manager.d.ts +18 -0
  87. package/dist/services/session-manager.d.ts.map +1 -0
  88. package/dist/services/session-manager.js +73 -0
  89. package/dist/services/session-manager.js.map +1 -0
  90. package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
  91. package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
  92. package/dist/strategies/exponential-backoff-strategy.js +32 -0
  93. package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
  94. package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
  95. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  96. package/dist/tools/handlers/fetch-links.tool.js +3 -0
  97. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  98. package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
  99. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  100. package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
  101. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  102. package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
  103. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  104. package/dist/tools/handlers/fetch-url.tool.js +15 -20
  105. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  106. package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
  107. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  108. package/dist/tools/handlers/fetch-urls.tool.js +141 -108
  109. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  110. package/dist/tools/index.d.ts.map +1 -1
  111. package/dist/tools/index.js +0 -4
  112. package/dist/tools/index.js.map +1 -1
  113. package/dist/tools/utils/common.d.ts +6 -7
  114. package/dist/tools/utils/common.d.ts.map +1 -1
  115. package/dist/tools/utils/common.js +8 -8
  116. package/dist/tools/utils/common.js.map +1 -1
  117. package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
  118. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  119. package/dist/tools/utils/fetch-pipeline.js +60 -63
  120. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  121. package/dist/transformers/jsonl.transformer.d.ts +1 -1
  122. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  123. package/dist/transformers/jsonl.transformer.js +15 -10
  124. package/dist/transformers/jsonl.transformer.js.map +1 -1
  125. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  126. package/dist/transformers/markdown.transformer.js +58 -62
  127. package/dist/transformers/markdown.transformer.js.map +1 -1
  128. package/dist/utils/concurrency.d.ts +2 -5
  129. package/dist/utils/concurrency.d.ts.map +1 -1
  130. package/dist/utils/concurrency.js +19 -19
  131. package/dist/utils/concurrency.js.map +1 -1
  132. package/dist/utils/content-cleaner.d.ts +0 -25
  133. package/dist/utils/content-cleaner.d.ts.map +1 -1
  134. package/dist/utils/content-cleaner.js +14 -171
  135. package/dist/utils/content-cleaner.js.map +1 -1
  136. package/dist/utils/html-truncator.d.ts +2 -0
  137. package/dist/utils/html-truncator.d.ts.map +1 -0
  138. package/dist/utils/html-truncator.js +14 -0
  139. package/dist/utils/html-truncator.js.map +1 -0
  140. package/dist/utils/language-detector.d.ts +0 -3
  141. package/dist/utils/language-detector.d.ts.map +1 -1
  142. package/dist/utils/language-detector.js +0 -11
  143. package/dist/utils/language-detector.js.map +1 -1
  144. package/dist/utils/sanitizer.d.ts.map +1 -1
  145. package/dist/utils/sanitizer.js +7 -5
  146. package/dist/utils/sanitizer.js.map +1 -1
  147. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  148. package/dist/utils/tool-error-handler.js +16 -41
  149. package/dist/utils/tool-error-handler.js.map +1 -1
  150. package/dist/utils/url-validator.d.ts +1 -0
  151. package/dist/utils/url-validator.d.ts.map +1 -1
  152. package/dist/utils/url-validator.js +42 -23
  153. package/dist/utils/url-validator.js.map +1 -1
  154. package/package.json +9 -8
package/README.md CHANGED
@@ -12,8 +12,6 @@
12
12
 
13
13
  A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
14
14
 
15
- **Version:** 1.0.4
16
-
17
15
  [Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
18
16
 
19
17
  > 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
@@ -108,7 +106,9 @@ Add to `.vscode/mcp.json` in your workspace:
108
106
  }
109
107
  ```
110
108
 
111
- ### With Environment Variables
109
+ ### With Custom Configuration
110
+
111
+ Configure SuperFetch behavior by adding environment variables to the `env` property:
112
112
 
113
113
  ```json
114
114
  {
@@ -118,13 +118,16 @@ Add to `.vscode/mcp.json` in your workspace:
118
118
  "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
119
119
  "env": {
120
120
  "CACHE_TTL": "7200",
121
- "LOG_LEVEL": "debug"
121
+ "LOG_LEVEL": "debug",
122
+ "FETCH_TIMEOUT": "60000"
122
123
  }
123
124
  }
124
125
  }
125
126
  }
126
127
  ```
127
128
 
129
+ See [Configuration](#configuration) section below for all available options and presets.
130
+
128
131
  ### Cursor
129
132
 
130
133
  1. Open Cursor Settings
@@ -456,25 +459,94 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
456
459
 
457
460
  ## Configuration
458
461
 
459
- ### Alternative MCP Client Setups
462
+ ### Configuration Presets
463
+
464
+ <details open>
465
+ <summary><strong>Default (Recommended)</strong> — No configuration needed</summary>
466
+
467
+ ```json
468
+ {
469
+ "servers": {
470
+ "superFetch": {
471
+ "command": "npx",
472
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
473
+ }
474
+ }
475
+ }
476
+ ```
477
+
478
+ </details>
460
479
 
461
480
  <details>
462
- <summary><strong>VS Code (HTTP mode)</strong> — requires running server separately</summary>
481
+ <summary><strong>Debug Mode</strong> — Verbose logging and no cache</summary>
463
482
 
464
- First, start the HTTP server:
483
+ **VS Code** (`.vscode/mcp.json`):
465
484
 
466
- ```bash
467
- npx -y @j0hanz/superfetch@latest
485
+ ```json
486
+ {
487
+ "servers": {
488
+ "superFetch": {
489
+ "command": "npx",
490
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
491
+ "env": {
492
+ "LOG_LEVEL": "debug",
493
+ "CACHE_ENABLED": "false"
494
+ }
495
+ }
496
+ }
497
+ }
498
+ ```
499
+
500
+ **Claude Desktop** (`claude_desktop_config.json`):
501
+
502
+ ```json
503
+ {
504
+ "mcpServers": {
505
+ "superFetch": {
506
+ "command": "npx",
507
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
508
+ "env": {
509
+ "LOG_LEVEL": "debug",
510
+ "CACHE_ENABLED": "false"
511
+ }
512
+ }
513
+ }
514
+ }
468
515
  ```
469
516
 
470
- Then add to `.vscode/mcp.json`:
517
+ **Cursor** (MCP settings):
518
+
519
+ ```json
520
+ {
521
+ "mcpServers": {
522
+ "superFetch": {
523
+ "command": "npx",
524
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
525
+ "env": {
526
+ "LOG_LEVEL": "debug",
527
+ "CACHE_ENABLED": "false"
528
+ }
529
+ }
530
+ }
531
+ }
532
+ ```
533
+
534
+ </details>
535
+
536
+ <details>
537
+ <summary><strong>Performance Mode</strong> — Aggressive caching for speed</summary>
471
538
 
472
539
  ```json
473
540
  {
474
541
  "servers": {
475
542
  "superFetch": {
476
- "type": "http",
477
- "url": "http://127.0.0.1:3000/mcp"
543
+ "command": "npx",
544
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
545
+ "env": {
546
+ "CACHE_TTL": "7200",
547
+ "CACHE_MAX_KEYS": "500",
548
+ "LOG_LEVEL": "warn"
549
+ }
478
550
  }
479
551
  }
480
552
  }
@@ -483,14 +555,17 @@ Then add to `.vscode/mcp.json`:
483
555
  </details>
484
556
 
485
557
  <details>
486
- <summary><strong>Claude Desktop (local path)</strong> — for development</summary>
558
+ <summary><strong>Custom User Agent</strong> — For sites that block bots</summary>
487
559
 
488
560
  ```json
489
561
  {
490
- "mcpServers": {
562
+ "servers": {
491
563
  "superFetch": {
492
- "command": "node",
493
- "args": ["/path/to/super-fetch-mcp-server/dist/index.js", "--stdio"]
564
+ "command": "npx",
565
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
566
+ "env": {
567
+ "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
568
+ }
494
569
  }
495
570
  }
496
571
  }
@@ -498,25 +573,114 @@ Then add to `.vscode/mcp.json`:
498
573
 
499
574
  </details>
500
575
 
501
- ### Environment Variables
502
-
503
- | Variable | Default | Description |
504
- | ---------------------- | -------------------- | ------------------------------- |
505
- | `PORT` | `3000` | HTTP server port |
506
- | `HOST` | `127.0.0.1` | HTTP server host |
507
- | `FETCH_TIMEOUT` | `30000` | Request timeout (ms) |
508
- | `MAX_REDIRECTS` | `5` | Maximum HTTP redirects |
509
- | `USER_AGENT` | `superFetch-MCP/1.0` | HTTP User-Agent |
510
- | `MAX_CONTENT_LENGTH` | `10485760` | Max response size (bytes) |
511
- | `CACHE_ENABLED` | `true` | Enable response caching |
512
- | `CACHE_TTL` | `3600` | Cache TTL (seconds) |
513
- | `CACHE_MAX_KEYS` | `100` | Maximum cache entries |
514
- | `LOG_LEVEL` | `info` | Logging level |
515
- | `ENABLE_LOGGING` | `true` | Enable/disable logging |
516
- | `EXTRACT_MAIN_CONTENT` | `true` | Extract main content by default |
517
- | `INCLUDE_METADATA` | `true` | Include metadata by default |
518
- | `MAX_BLOCK_LENGTH` | `5000` | Maximum block length |
519
- | `MIN_PARAGRAPH_LENGTH` | `10` | Minimum paragraph length |
576
+ <details>
577
+ <summary><strong>Slow Networks / CI/CD</strong> — Extended timeouts</summary>
578
+
579
+ ```json
580
+ {
581
+ "servers": {
582
+ "superFetch": {
583
+ "command": "npx",
584
+ "args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
585
+ "env": {
586
+ "FETCH_TIMEOUT": "60000",
587
+ "CACHE_ENABLED": "false",
588
+ "LOG_LEVEL": "warn"
589
+ }
590
+ }
591
+ }
592
+ }
593
+ ```
594
+
595
+ </details>
596
+
597
+ ### Available Environment Variables
598
+
599
+ Configure SuperFetch behavior by adding environment variables to your MCP client configuration's `env` property.
600
+
601
+ #### 🌐 Fetcher Settings
602
+
603
+ | Variable | Default | Valid Values | Description |
604
+ | --------------- | -------------------- | -------------------- | --------------------------------------------------------------- |
605
+ | `FETCH_TIMEOUT` | `30000` | `5000`-`120000` | Request timeout in milliseconds (5s-2min) |
606
+ | `USER_AGENT` | `superFetch-MCP/1.0` | Any valid user agent | Custom user agent for requests (useful for sites blocking bots) |
607
+
608
+ #### 💾 Cache Settings
609
+
610
+ | Variable | Default | Valid Values | Description |
611
+ | ---------------- | ------- | ---------------- | -------------------------------------- |
612
+ | `CACHE_ENABLED` | `true` | `true` / `false` | Enable response caching |
613
+ | `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
614
+ | `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
615
+
616
+ #### 📝 Logging Settings
617
+
618
+ | Variable | Default | Valid Values | Description |
619
+ | ---------------- | ------- | ----------------------------------- | -------------------------- |
620
+ | `LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error` | Logging verbosity level |
621
+ | `ENABLE_LOGGING` | `true` | `true` / `false` | Enable/disable all logging |
622
+
623
+ #### 🔍 Extraction Settings
624
+
625
+ | Variable | Default | Valid Values | Description |
626
+ | ---------------------- | ------- | ---------------- | -------------------------------------------------- |
627
+ | `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
628
+ | `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
629
+
630
+ ### HTTP Mode Configuration
631
+
632
+ <details>
633
+ <summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
634
+
635
+ SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
636
+
637
+ #### Start HTTP Server
638
+
639
+ ```bash
640
+ npx -y @j0hanz/superfetch@latest
641
+ # Server runs at http://127.0.0.1:3000
642
+ ```
643
+
644
+ #### HTTP-Specific Environment Variables
645
+
646
+ | Variable | Default | Description |
647
+ | ----------------- | ----------- | ------------------------------------------------ |
648
+ | `PORT` | `3000` | HTTP server port |
649
+ | `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
650
+ | `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
651
+ | `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
652
+
653
+ #### VS Code HTTP Mode Setup
654
+
655
+ ```json
656
+ {
657
+ "servers": {
658
+ "superFetch": {
659
+ "type": "http",
660
+ "url": "http://127.0.0.1:3000/mcp"
661
+ }
662
+ }
663
+ }
664
+ ```
665
+
666
+ #### Docker/Kubernetes Example
667
+
668
+ ```bash
669
+ PORT=8080 HOST=0.0.0.0 ALLOWED_ORIGINS=https://myapp.com npx @j0hanz/superfetch@latest
670
+ ```
671
+
672
+ </details>
673
+
674
+ ### Configuration Cookbook
675
+
676
+ | Use Case | Configuration |
677
+ | ---------------------------- | -------------------------------------------------------------- |
678
+ | 🐛 **Debugging issues** | `LOG_LEVEL=debug`, `CACHE_ENABLED=false` |
679
+ | 🚀 **Maximum performance** | `CACHE_TTL=7200`, `CACHE_MAX_KEYS=500`, `LOG_LEVEL=error` |
680
+ | 🌐 **Slow target sites** | `FETCH_TIMEOUT=60000` |
681
+ | 🤖 **Bypass bot detection** | `USER_AGENT="Mozilla/5.0 (compatible; MyBot/1.0)"` |
682
+ | 🔄 **CI/CD (always fresh)** | `CACHE_ENABLED=false`, `FETCH_TIMEOUT=60000`, `LOG_LEVEL=warn` |
683
+ | 📊 **Production monitoring** | `LOG_LEVEL=warn` or `error` |
520
684
 
521
685
  ---
522
686
 
@@ -592,7 +756,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
592
756
  | ------------------ | --------------------------------- |
593
757
  | Runtime | Node.js ≥20.0.0 |
594
758
  | Language | TypeScript 5.9 |
595
- | MCP SDK | @modelcontextprotocol/sdk ^1.24.3 |
759
+ | MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
596
760
  | Content Extraction | @mozilla/readability ^0.6.0 |
597
761
  | HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
598
762
  | Markdown | Turndown ^7.2.2 |
@@ -7,9 +7,9 @@ export declare const config: {
7
7
  };
8
8
  readonly fetcher: {
9
9
  readonly timeout: number;
10
- readonly maxRedirects: number;
10
+ readonly maxRedirects: 5;
11
11
  readonly userAgent: string;
12
- readonly maxContentLength: number;
12
+ readonly maxContentLength: 10485760;
13
13
  };
14
14
  readonly cache: {
15
15
  readonly enabled: boolean;
@@ -19,12 +19,17 @@ export declare const config: {
19
19
  readonly extraction: {
20
20
  readonly extractMainContent: boolean;
21
21
  readonly includeMetadata: boolean;
22
- readonly maxBlockLength: number;
23
- readonly minParagraphLength: number;
22
+ readonly maxBlockLength: 5000;
23
+ readonly minParagraphLength: 10;
24
24
  };
25
25
  readonly logging: {
26
- readonly level: string;
26
+ readonly level: "debug" | "info" | "warn" | "error";
27
27
  readonly enabled: boolean;
28
28
  };
29
+ readonly constants: {
30
+ readonly maxHtmlSize: number;
31
+ readonly maxContentSize: number;
32
+ readonly maxUrlLength: 2048;
33
+ };
29
34
  };
30
35
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsCT,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAqCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiCT,CAAC"}
@@ -1,38 +1,62 @@
1
- function parseIntEnv(value, defaultValue, min = 0, max = Number.MAX_SAFE_INTEGER) {
2
- if (!value)
1
+ function parseInteger(envValue, defaultValue, min, max) {
2
+ if (!envValue)
3
3
  return defaultValue;
4
- const parsed = parseInt(value, 10);
4
+ const parsed = parseInt(envValue, 10);
5
5
  if (isNaN(parsed))
6
6
  return defaultValue;
7
- return Math.max(min, Math.min(max, parsed));
7
+ if (min !== undefined && parsed < min)
8
+ return defaultValue;
9
+ if (max !== undefined && parsed > max)
10
+ return defaultValue;
11
+ return parsed;
12
+ }
13
+ function parseBoolean(envValue, defaultValue) {
14
+ if (!envValue)
15
+ return defaultValue;
16
+ return envValue !== 'false';
17
+ }
18
+ function parseLogLevel(envValue) {
19
+ const level = envValue?.toLowerCase();
20
+ if (level === 'debug' ||
21
+ level === 'info' ||
22
+ level === 'warn' ||
23
+ level === 'error') {
24
+ return level;
25
+ }
26
+ return 'info';
8
27
  }
9
28
  export const config = {
10
29
  server: {
11
30
  name: 'superFetch',
12
31
  version: '1.0.0',
13
- port: parseIntEnv(process.env.PORT, 3000, 1, 65535),
32
+ port: parseInteger(process.env.PORT, 3000, 1024, 65535),
14
33
  host: process.env.HOST ?? '127.0.0.1',
15
34
  },
16
35
  fetcher: {
17
- timeout: parseIntEnv(process.env.FETCH_TIMEOUT, 30000, 1000, 120000),
18
- maxRedirects: parseIntEnv(process.env.MAX_REDIRECTS, 5, 0, 20),
36
+ timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
37
+ maxRedirects: 5,
19
38
  userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
20
- maxContentLength: parseIntEnv(process.env.MAX_CONTENT_LENGTH, 10485760, 1024, 52428800),
39
+ maxContentLength: 10485760,
21
40
  },
22
41
  cache: {
23
- enabled: process.env.CACHE_ENABLED !== 'false',
24
- ttl: parseIntEnv(process.env.CACHE_TTL, 3600, 60, 86400),
25
- maxKeys: parseIntEnv(process.env.CACHE_MAX_KEYS, 100, 10, 10000),
42
+ enabled: parseBoolean(process.env.CACHE_ENABLED, true),
43
+ ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
44
+ maxKeys: parseInteger(process.env.CACHE_MAX_KEYS, 100, 10, 1000),
26
45
  },
27
46
  extraction: {
28
- extractMainContent: process.env.EXTRACT_MAIN_CONTENT !== 'false',
29
- includeMetadata: process.env.INCLUDE_METADATA !== 'false',
30
- maxBlockLength: parseIntEnv(process.env.MAX_BLOCK_LENGTH, 5000, 100, 50000),
31
- minParagraphLength: parseIntEnv(process.env.MIN_PARAGRAPH_LENGTH, 10, 0, 1000),
47
+ extractMainContent: parseBoolean(process.env.EXTRACT_MAIN_CONTENT, true),
48
+ includeMetadata: parseBoolean(process.env.INCLUDE_METADATA, true),
49
+ maxBlockLength: 5000,
50
+ minParagraphLength: 10,
32
51
  },
33
52
  logging: {
34
- level: process.env.LOG_LEVEL ?? 'info',
35
- enabled: process.env.ENABLE_LOGGING !== 'false',
53
+ level: parseLogLevel(process.env.LOG_LEVEL),
54
+ enabled: parseBoolean(process.env.ENABLE_LOGGING, true),
55
+ },
56
+ constants: {
57
+ maxHtmlSize: 10 * 1024 * 1024,
58
+ maxContentSize: 5 * 1024 * 1024,
59
+ maxUrlLength: 2048,
36
60
  },
37
61
  };
38
62
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,WAAW,CAClB,KAAyB,EACzB,YAAoB,EACpB,GAAG,GAAG,CAAC,EACP,GAAG,GAAG,MAAM,CAAC,gBAAgB;IAE7B,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC;QACnD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACpE,YAAY,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9D,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,WAAW,CAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAC9B,QAAQ,EACR,IAAI,EACJ,QAAQ,CACT;KACF;IACD,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,KAAK,OAAO;QAC9C,GAAG,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO;QAChE,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,OAAO;QACzD,cAAc,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC;QAC3E,kBAAkB,EAAE,WAAW,CAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAChC,EAAE,EACF,CAAC,EACD,IAAI,CACL;KACF;IACD,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO;KAChD;CACO,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;CACO,CAAC"}