@j0hanz/superfetch 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -36
- package/dist/config/index.d.ts +5 -5
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +38 -19
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +98 -57
- package/dist/config/types.d.ts.map +1 -1
- package/dist/errors/app-error.d.ts +4 -28
- package/dist/errors/app-error.d.ts.map +1 -1
- package/dist/errors/app-error.js +10 -51
- package/dist/errors/app-error.js.map +1 -1
- package/dist/index.js +10 -55
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +12 -14
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/middleware/rate-limiter.d.ts.map +1 -1
- package/dist/middleware/rate-limiter.js +0 -8
- package/dist/middleware/rate-limiter.js.map +1 -1
- package/dist/parsers/base-html-element-parser.d.ts +43 -0
- package/dist/parsers/base-html-element-parser.d.ts.map +1 -0
- package/dist/parsers/base-html-element-parser.js +59 -0
- package/dist/parsers/base-html-element-parser.js.map +1 -0
- package/dist/parsers/heading-element-parser.d.ts +14 -0
- package/dist/parsers/heading-element-parser.d.ts.map +1 -0
- package/dist/parsers/heading-element-parser.js +26 -0
- package/dist/parsers/heading-element-parser.js.map +1 -0
- package/dist/parsers/image-element-parser.d.ts +16 -0
- package/dist/parsers/image-element-parser.d.ts.map +1 -0
- package/dist/parsers/image-element-parser.js +33 -0
- package/dist/parsers/image-element-parser.js.map +1 -0
- package/dist/parsers/link-element-parser.d.ts +15 -0
- package/dist/parsers/link-element-parser.d.ts.map +1 -0
- package/dist/parsers/link-element-parser.js +28 -0
- package/dist/parsers/link-element-parser.js.map +1 -0
- package/dist/parsers/open-graph-parser.d.ts +17 -0
- package/dist/parsers/open-graph-parser.d.ts.map +1 -0
- package/dist/parsers/open-graph-parser.js +41 -0
- package/dist/parsers/open-graph-parser.js.map +1 -0
- package/dist/parsers/schema-org-parser.d.ts +17 -0
- package/dist/parsers/schema-org-parser.d.ts.map +1 -0
- package/dist/parsers/schema-org-parser.js +32 -0
- package/dist/parsers/schema-org-parser.js.map +1 -0
- package/dist/parsers/standard-meta-parser.d.ts +18 -0
- package/dist/parsers/standard-meta-parser.d.ts.map +1 -0
- package/dist/parsers/standard-meta-parser.js +32 -0
- package/dist/parsers/standard-meta-parser.js.map +1 -0
- package/dist/parsers/twitter-card-parser.d.ts +17 -0
- package/dist/parsers/twitter-card-parser.d.ts.map +1 -0
- package/dist/parsers/twitter-card-parser.js +41 -0
- package/dist/parsers/twitter-card-parser.js.map +1 -0
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +3 -9
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +8 -8
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +10 -10
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +0 -28
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +10 -173
- package/dist/services/cache.js.map +1 -1
- package/dist/services/extractor.d.ts +1 -11
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +86 -84
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher.d.ts +2 -13
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +48 -79
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +5 -4
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +27 -42
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +35 -26
- package/dist/services/parser.js.map +1 -1
- package/dist/services/session-manager.d.ts +18 -0
- package/dist/services/session-manager.d.ts.map +1 -0
- package/dist/services/session-manager.js +73 -0
- package/dist/services/session-manager.js.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts +13 -0
- package/dist/strategies/exponential-backoff-strategy.d.ts.map +1 -0
- package/dist/strategies/exponential-backoff-strategy.js +32 -0
- package/dist/strategies/exponential-backoff-strategy.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -1
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +5 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +23 -33
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +15 -20
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.d.ts +2 -9
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +124 -105
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +0 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/utils/common.d.ts +6 -7
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +8 -8
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +8 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +47 -79
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -1
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +15 -10
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +58 -62
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/concurrency.d.ts +2 -5
- package/dist/utils/concurrency.d.ts.map +1 -1
- package/dist/utils/concurrency.js +19 -19
- package/dist/utils/concurrency.js.map +1 -1
- package/dist/utils/content-cleaner.d.ts +0 -25
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +12 -187
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +2 -0
- package/dist/utils/html-truncator.d.ts.map +1 -0
- package/dist/utils/html-truncator.js +14 -0
- package/dist/utils/html-truncator.js.map +1 -0
- package/dist/utils/language-detector.d.ts +0 -3
- package/dist/utils/language-detector.d.ts.map +1 -1
- package/dist/utils/language-detector.js +0 -11
- package/dist/utils/language-detector.js.map +1 -1
- package/dist/utils/sanitizer.d.ts.map +1 -1
- package/dist/utils/sanitizer.js +7 -5
- package/dist/utils/sanitizer.js.map +1 -1
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +15 -42
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -9
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +14 -33
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +3 -4
package/README.md
CHANGED
|
@@ -12,8 +12,6 @@
|
|
|
12
12
|
|
|
13
13
|
A [Model Context Protocol](https://modelcontextprotocol.io/) (MCP) server that fetches, extracts, and transforms web content into AI-optimized formats using Mozilla Readability.
|
|
14
14
|
|
|
15
|
-
**Version:** 1.0.5
|
|
16
|
-
|
|
17
15
|
[Quick Start](#quick-start) · [How to Choose a Tool](#-how-to-choose-a-tool) · [Tools](#available-tools) · [Configuration](#configuration) · [Contributing](#contributing)
|
|
18
16
|
|
|
19
17
|
> 📦 **Published to [MCP Registry](https://registry.modelcontextprotocol.io/)** — Search for `io.github.j0hanz/superfetch`
|
|
@@ -108,7 +106,9 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
108
106
|
}
|
|
109
107
|
```
|
|
110
108
|
|
|
111
|
-
### With
|
|
109
|
+
### With Custom Configuration
|
|
110
|
+
|
|
111
|
+
Configure SuperFetch behavior by adding environment variables to the `env` property:
|
|
112
112
|
|
|
113
113
|
```json
|
|
114
114
|
{
|
|
@@ -118,13 +118,16 @@ Add to `.vscode/mcp.json` in your workspace:
|
|
|
118
118
|
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
119
119
|
"env": {
|
|
120
120
|
"CACHE_TTL": "7200",
|
|
121
|
-
"LOG_LEVEL": "debug"
|
|
121
|
+
"LOG_LEVEL": "debug",
|
|
122
|
+
"FETCH_TIMEOUT": "60000"
|
|
122
123
|
}
|
|
123
124
|
}
|
|
124
125
|
}
|
|
125
126
|
}
|
|
126
127
|
```
|
|
127
128
|
|
|
129
|
+
See [Configuration](#configuration) section below for all available options and presets.
|
|
130
|
+
|
|
128
131
|
### Cursor
|
|
129
132
|
|
|
130
133
|
1. Open Cursor Settings
|
|
@@ -456,25 +459,94 @@ Fetches multiple URLs in parallel with concurrency control. Ideal for comparing
|
|
|
456
459
|
|
|
457
460
|
## Configuration
|
|
458
461
|
|
|
459
|
-
###
|
|
462
|
+
### Configuration Presets
|
|
463
|
+
|
|
464
|
+
<details open>
|
|
465
|
+
<summary><strong>Default (Recommended)</strong> — No configuration needed</summary>
|
|
466
|
+
|
|
467
|
+
```json
|
|
468
|
+
{
|
|
469
|
+
"servers": {
|
|
470
|
+
"superFetch": {
|
|
471
|
+
"command": "npx",
|
|
472
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"]
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
```
|
|
477
|
+
|
|
478
|
+
</details>
|
|
460
479
|
|
|
461
480
|
<details>
|
|
462
|
-
<summary><strong>
|
|
481
|
+
<summary><strong>Debug Mode</strong> — Verbose logging and no cache</summary>
|
|
463
482
|
|
|
464
|
-
|
|
483
|
+
**VS Code** (`.vscode/mcp.json`):
|
|
465
484
|
|
|
466
|
-
```
|
|
467
|
-
|
|
485
|
+
```json
|
|
486
|
+
{
|
|
487
|
+
"servers": {
|
|
488
|
+
"superFetch": {
|
|
489
|
+
"command": "npx",
|
|
490
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
491
|
+
"env": {
|
|
492
|
+
"LOG_LEVEL": "debug",
|
|
493
|
+
"CACHE_ENABLED": "false"
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
**Claude Desktop** (`claude_desktop_config.json`):
|
|
501
|
+
|
|
502
|
+
```json
|
|
503
|
+
{
|
|
504
|
+
"mcpServers": {
|
|
505
|
+
"superFetch": {
|
|
506
|
+
"command": "npx",
|
|
507
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
508
|
+
"env": {
|
|
509
|
+
"LOG_LEVEL": "debug",
|
|
510
|
+
"CACHE_ENABLED": "false"
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
468
515
|
```
|
|
469
516
|
|
|
470
|
-
|
|
517
|
+
**Cursor** (MCP settings):
|
|
518
|
+
|
|
519
|
+
```json
|
|
520
|
+
{
|
|
521
|
+
"mcpServers": {
|
|
522
|
+
"superFetch": {
|
|
523
|
+
"command": "npx",
|
|
524
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
525
|
+
"env": {
|
|
526
|
+
"LOG_LEVEL": "debug",
|
|
527
|
+
"CACHE_ENABLED": "false"
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
</details>
|
|
535
|
+
|
|
536
|
+
<details>
|
|
537
|
+
<summary><strong>Performance Mode</strong> — Aggressive caching for speed</summary>
|
|
471
538
|
|
|
472
539
|
```json
|
|
473
540
|
{
|
|
474
541
|
"servers": {
|
|
475
542
|
"superFetch": {
|
|
476
|
-
"
|
|
477
|
-
"
|
|
543
|
+
"command": "npx",
|
|
544
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
545
|
+
"env": {
|
|
546
|
+
"CACHE_TTL": "7200",
|
|
547
|
+
"CACHE_MAX_KEYS": "500",
|
|
548
|
+
"LOG_LEVEL": "warn"
|
|
549
|
+
}
|
|
478
550
|
}
|
|
479
551
|
}
|
|
480
552
|
}
|
|
@@ -483,14 +555,17 @@ Then add to `.vscode/mcp.json`:
|
|
|
483
555
|
</details>
|
|
484
556
|
|
|
485
557
|
<details>
|
|
486
|
-
<summary><strong>
|
|
558
|
+
<summary><strong>Custom User Agent</strong> — For sites that block bots</summary>
|
|
487
559
|
|
|
488
560
|
```json
|
|
489
561
|
{
|
|
490
|
-
"
|
|
562
|
+
"servers": {
|
|
491
563
|
"superFetch": {
|
|
492
|
-
"command": "
|
|
493
|
-
"args": ["
|
|
564
|
+
"command": "npx",
|
|
565
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
566
|
+
"env": {
|
|
567
|
+
"USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
568
|
+
}
|
|
494
569
|
}
|
|
495
570
|
}
|
|
496
571
|
}
|
|
@@ -498,25 +573,114 @@ Then add to `.vscode/mcp.json`:
|
|
|
498
573
|
|
|
499
574
|
</details>
|
|
500
575
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
576
|
+
<details>
|
|
577
|
+
<summary><strong>Slow Networks / CI/CD</strong> — Extended timeouts</summary>
|
|
578
|
+
|
|
579
|
+
```json
|
|
580
|
+
{
|
|
581
|
+
"servers": {
|
|
582
|
+
"superFetch": {
|
|
583
|
+
"command": "npx",
|
|
584
|
+
"args": ["-y", "@j0hanz/superfetch@latest", "--stdio"],
|
|
585
|
+
"env": {
|
|
586
|
+
"FETCH_TIMEOUT": "60000",
|
|
587
|
+
"CACHE_ENABLED": "false",
|
|
588
|
+
"LOG_LEVEL": "warn"
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
```
|
|
594
|
+
|
|
595
|
+
</details>
|
|
596
|
+
|
|
597
|
+
### Available Environment Variables
|
|
598
|
+
|
|
599
|
+
Configure SuperFetch behavior by adding environment variables to your MCP client configuration's `env` property.
|
|
600
|
+
|
|
601
|
+
#### 🌐 Fetcher Settings
|
|
602
|
+
|
|
603
|
+
| Variable | Default | Valid Values | Description |
|
|
604
|
+
| --------------- | -------------------- | -------------------- | --------------------------------------------------------------- |
|
|
605
|
+
| `FETCH_TIMEOUT` | `30000` | `5000`-`120000` | Request timeout in milliseconds (5s-2min) |
|
|
606
|
+
| `USER_AGENT` | `superFetch-MCP/1.0` | Any valid user agent | Custom user agent for requests (useful for sites blocking bots) |
|
|
607
|
+
|
|
608
|
+
#### 💾 Cache Settings
|
|
609
|
+
|
|
610
|
+
| Variable | Default | Valid Values | Description |
|
|
611
|
+
| ---------------- | ------- | ---------------- | -------------------------------------- |
|
|
612
|
+
| `CACHE_ENABLED` | `true` | `true` / `false` | Enable response caching |
|
|
613
|
+
| `CACHE_TTL` | `3600` | `60`-`86400` | Cache lifetime in seconds (1min-24hrs) |
|
|
614
|
+
| `CACHE_MAX_KEYS` | `100` | `10`-`1000` | Maximum number of cached entries |
|
|
615
|
+
|
|
616
|
+
#### 📝 Logging Settings
|
|
617
|
+
|
|
618
|
+
| Variable | Default | Valid Values | Description |
|
|
619
|
+
| ---------------- | ------- | ----------------------------------- | -------------------------- |
|
|
620
|
+
| `LOG_LEVEL` | `info` | `debug` / `info` / `warn` / `error` | Logging verbosity level |
|
|
621
|
+
| `ENABLE_LOGGING` | `true` | `true` / `false` | Enable/disable all logging |
|
|
622
|
+
|
|
623
|
+
#### 🔍 Extraction Settings
|
|
624
|
+
|
|
625
|
+
| Variable | Default | Valid Values | Description |
|
|
626
|
+
| ---------------------- | ------- | ---------------- | -------------------------------------------------- |
|
|
627
|
+
| `EXTRACT_MAIN_CONTENT` | `true` | `true` / `false` | Use Mozilla Readability to extract main content |
|
|
628
|
+
| `INCLUDE_METADATA` | `true` | `true` / `false` | Include page metadata (title, description, author) |
|
|
629
|
+
|
|
630
|
+
### HTTP Mode Configuration
|
|
631
|
+
|
|
632
|
+
<details>
|
|
633
|
+
<summary><strong>HTTP Mode</strong> (Advanced) — For running as a standalone HTTP server</summary>
|
|
634
|
+
|
|
635
|
+
SuperFetch can run as an HTTP server for custom integrations. HTTP mode requires additional configuration:
|
|
636
|
+
|
|
637
|
+
#### Start HTTP Server
|
|
638
|
+
|
|
639
|
+
```bash
|
|
640
|
+
npx -y @j0hanz/superfetch@latest
|
|
641
|
+
# Server runs at http://127.0.0.1:3000
|
|
642
|
+
```
|
|
643
|
+
|
|
644
|
+
#### HTTP-Specific Environment Variables
|
|
645
|
+
|
|
646
|
+
| Variable | Default | Description |
|
|
647
|
+
| ----------------- | ----------- | ------------------------------------------------ |
|
|
648
|
+
| `PORT` | `3000` | HTTP server port |
|
|
649
|
+
| `HOST` | `127.0.0.1` | HTTP server host (`0.0.0.0` for Docker/K8s) |
|
|
650
|
+
| `ALLOWED_ORIGINS` | `[]` | Comma-separated CORS origins |
|
|
651
|
+
| `CORS_ALLOW_ALL` | `false` | Allow all CORS origins (dev only, security risk) |
|
|
652
|
+
|
|
653
|
+
#### VS Code HTTP Mode Setup
|
|
654
|
+
|
|
655
|
+
```json
|
|
656
|
+
{
|
|
657
|
+
"servers": {
|
|
658
|
+
"superFetch": {
|
|
659
|
+
"type": "http",
|
|
660
|
+
"url": "http://127.0.0.1:3000/mcp"
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
```
|
|
665
|
+
|
|
666
|
+
#### Docker/Kubernetes Example
|
|
667
|
+
|
|
668
|
+
```bash
|
|
669
|
+
PORT=8080 HOST=0.0.0.0 ALLOWED_ORIGINS=https://myapp.com npx @j0hanz/superfetch@latest
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
</details>
|
|
673
|
+
|
|
674
|
+
### Configuration Cookbook
|
|
675
|
+
|
|
676
|
+
| Use Case | Configuration |
|
|
677
|
+
| ---------------------------- | -------------------------------------------------------------- |
|
|
678
|
+
| 🐛 **Debugging issues** | `LOG_LEVEL=debug`, `CACHE_ENABLED=false` |
|
|
679
|
+
| 🚀 **Maximum performance** | `CACHE_TTL=7200`, `CACHE_MAX_KEYS=500`, `LOG_LEVEL=error` |
|
|
680
|
+
| 🌐 **Slow target sites** | `FETCH_TIMEOUT=60000` |
|
|
681
|
+
| 🤖 **Bypass bot detection** | `USER_AGENT="Mozilla/5.0 (compatible; MyBot/1.0)"` |
|
|
682
|
+
| 🔄 **CI/CD (always fresh)** | `CACHE_ENABLED=false`, `FETCH_TIMEOUT=60000`, `LOG_LEVEL=warn` |
|
|
683
|
+
| 📊 **Production monitoring** | `LOG_LEVEL=warn` or `error` |
|
|
520
684
|
|
|
521
685
|
---
|
|
522
686
|
|
|
@@ -592,7 +756,7 @@ Sessions are managed via `mcp-session-id` header with 30-minute TTL.
|
|
|
592
756
|
| ------------------ | --------------------------------- |
|
|
593
757
|
| Runtime | Node.js ≥20.0.0 |
|
|
594
758
|
| Language | TypeScript 5.9 |
|
|
595
|
-
| MCP SDK | @modelcontextprotocol/sdk ^1.
|
|
759
|
+
| MCP SDK | @modelcontextprotocol/sdk ^1.25.1 |
|
|
596
760
|
| Content Extraction | @mozilla/readability ^0.6.0 |
|
|
597
761
|
| HTML Parsing | Cheerio ^1.1.2, JSDOM ^27.3.0 |
|
|
598
762
|
| Markdown | Turndown ^7.2.2 |
|
package/dist/config/index.d.ts
CHANGED
|
@@ -7,9 +7,9 @@ export declare const config: {
|
|
|
7
7
|
};
|
|
8
8
|
readonly fetcher: {
|
|
9
9
|
readonly timeout: number;
|
|
10
|
-
readonly maxRedirects:
|
|
10
|
+
readonly maxRedirects: 5;
|
|
11
11
|
readonly userAgent: string;
|
|
12
|
-
readonly maxContentLength:
|
|
12
|
+
readonly maxContentLength: 10485760;
|
|
13
13
|
};
|
|
14
14
|
readonly cache: {
|
|
15
15
|
readonly enabled: boolean;
|
|
@@ -19,11 +19,11 @@ export declare const config: {
|
|
|
19
19
|
readonly extraction: {
|
|
20
20
|
readonly extractMainContent: boolean;
|
|
21
21
|
readonly includeMetadata: boolean;
|
|
22
|
-
readonly maxBlockLength:
|
|
23
|
-
readonly minParagraphLength:
|
|
22
|
+
readonly maxBlockLength: 5000;
|
|
23
|
+
readonly minParagraphLength: 10;
|
|
24
24
|
};
|
|
25
25
|
readonly logging: {
|
|
26
|
-
readonly level:
|
|
26
|
+
readonly level: "debug" | "info" | "warn" | "error";
|
|
27
27
|
readonly enabled: boolean;
|
|
28
28
|
};
|
|
29
29
|
readonly constants: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAqCA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiCT,CAAC"}
|
package/dist/config/index.js
CHANGED
|
@@ -1,42 +1,61 @@
|
|
|
1
|
-
function
|
|
2
|
-
if (!
|
|
1
|
+
function parseInteger(envValue, defaultValue, min, max) {
|
|
2
|
+
if (!envValue)
|
|
3
3
|
return defaultValue;
|
|
4
|
-
const parsed = parseInt(
|
|
4
|
+
const parsed = parseInt(envValue, 10);
|
|
5
5
|
if (isNaN(parsed))
|
|
6
6
|
return defaultValue;
|
|
7
|
-
|
|
7
|
+
if (min !== undefined && parsed < min)
|
|
8
|
+
return defaultValue;
|
|
9
|
+
if (max !== undefined && parsed > max)
|
|
10
|
+
return defaultValue;
|
|
11
|
+
return parsed;
|
|
12
|
+
}
|
|
13
|
+
function parseBoolean(envValue, defaultValue) {
|
|
14
|
+
if (!envValue)
|
|
15
|
+
return defaultValue;
|
|
16
|
+
return envValue !== 'false';
|
|
17
|
+
}
|
|
18
|
+
function parseLogLevel(envValue) {
|
|
19
|
+
const level = envValue?.toLowerCase();
|
|
20
|
+
if (level === 'debug' ||
|
|
21
|
+
level === 'info' ||
|
|
22
|
+
level === 'warn' ||
|
|
23
|
+
level === 'error') {
|
|
24
|
+
return level;
|
|
25
|
+
}
|
|
26
|
+
return 'info';
|
|
8
27
|
}
|
|
9
28
|
export const config = {
|
|
10
29
|
server: {
|
|
11
30
|
name: 'superFetch',
|
|
12
31
|
version: '1.0.0',
|
|
13
|
-
port:
|
|
32
|
+
port: parseInteger(process.env.PORT, 3000, 1024, 65535),
|
|
14
33
|
host: process.env.HOST ?? '127.0.0.1',
|
|
15
34
|
},
|
|
16
35
|
fetcher: {
|
|
17
|
-
timeout:
|
|
18
|
-
maxRedirects:
|
|
36
|
+
timeout: parseInteger(process.env.FETCH_TIMEOUT, 30000, 5000, 120000),
|
|
37
|
+
maxRedirects: 5,
|
|
19
38
|
userAgent: process.env.USER_AGENT ?? 'superFetch-MCP/1.0',
|
|
20
|
-
maxContentLength:
|
|
39
|
+
maxContentLength: 10485760,
|
|
21
40
|
},
|
|
22
41
|
cache: {
|
|
23
|
-
enabled: process.env.CACHE_ENABLED
|
|
24
|
-
ttl:
|
|
25
|
-
maxKeys:
|
|
42
|
+
enabled: parseBoolean(process.env.CACHE_ENABLED, true),
|
|
43
|
+
ttl: parseInteger(process.env.CACHE_TTL, 3600, 60, 86400),
|
|
44
|
+
maxKeys: parseInteger(process.env.CACHE_MAX_KEYS, 100, 10, 1000),
|
|
26
45
|
},
|
|
27
46
|
extraction: {
|
|
28
|
-
extractMainContent: process.env.EXTRACT_MAIN_CONTENT
|
|
29
|
-
includeMetadata: process.env.INCLUDE_METADATA
|
|
30
|
-
maxBlockLength:
|
|
31
|
-
minParagraphLength:
|
|
47
|
+
extractMainContent: parseBoolean(process.env.EXTRACT_MAIN_CONTENT, true),
|
|
48
|
+
includeMetadata: parseBoolean(process.env.INCLUDE_METADATA, true),
|
|
49
|
+
maxBlockLength: 5000,
|
|
50
|
+
minParagraphLength: 10,
|
|
32
51
|
},
|
|
33
52
|
logging: {
|
|
34
|
-
level: process.env.LOG_LEVEL
|
|
35
|
-
enabled: process.env.ENABLE_LOGGING
|
|
53
|
+
level: parseLogLevel(process.env.LOG_LEVEL),
|
|
54
|
+
enabled: parseBoolean(process.env.ENABLE_LOGGING, true),
|
|
36
55
|
},
|
|
37
56
|
constants: {
|
|
38
|
-
maxHtmlSize: 10 * 1024 * 1024,
|
|
39
|
-
maxContentSize: 5 * 1024 * 1024,
|
|
57
|
+
maxHtmlSize: 10 * 1024 * 1024,
|
|
58
|
+
maxContentSize: 5 * 1024 * 1024,
|
|
40
59
|
maxUrlLength: 2048,
|
|
41
60
|
},
|
|
42
61
|
};
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAoB,EACpB,GAAY,EACZ,GAAY;IAEZ,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,QAA4B,EAC5B,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,YAAY,CAAC;IACnC,OAAO,QAAQ,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CACpB,QAA4B;IAE5B,MAAM,KAAK,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC;IACtC,IACE,KAAK,KAAK,OAAO;QACjB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,MAAM;QAChB,KAAK,KAAK,OAAO,EACjB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,MAAM,EAAE;QACN,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QACvD,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW;KACtC;IACD,OAAO,EAAE;QACP,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;QACrE,YAAY,EAAE,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,oBAAoB;QACzD,gBAAgB,EAAE,QAAQ;KAC3B;IACD,KAAK,EAAE;QACL,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC;QACtD,GAAG,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC;QACzD,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;KACjE;IACD,UAAU,EAAE;QACV,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACxE,eAAe,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC;QACjE,cAAc,EAAE,IAAI;QACpB,kBAAkB,EAAE,EAAE;KACvB;IACD,OAAO,EAAE;QACP,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC3C,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC;KACxD;IACD,SAAS,EAAE;QACT,WAAW,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC7B,cAAc,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QAC/B,YAAY,EAAE,IAAI;KACnB;CACO,CAAC"}
|