@j0hanz/fetch-url-mcp 1.11.7 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +177 -99
- package/dist/http/health.d.ts.map +1 -1
- package/dist/http/health.js +0 -2
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +0 -25
- package/dist/lib/config.d.ts +0 -7
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +0 -9
- package/dist/lib/core.d.ts +0 -3
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +0 -7
- package/dist/lib/fetch-pipeline.d.ts +1 -14
- package/dist/lib/fetch-pipeline.d.ts.map +1 -1
- package/dist/lib/fetch-pipeline.js +4 -147
- package/dist/lib/http.d.ts +0 -3
- package/dist/lib/http.d.ts.map +1 -1
- package/dist/lib/http.js +2 -105
- package/dist/lib/mcp-interop.js +2 -2
- package/dist/lib/utils.d.ts +0 -2
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +4 -39
- package/dist/resources/index.d.ts +1 -23
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +3 -294
- package/dist/schemas.d.ts +0 -14
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +1 -77
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +1 -2
- package/dist/tools/fetch-url.d.ts +0 -2
- package/dist/tools/fetch-url.d.ts.map +1 -1
- package/dist/tools/fetch-url.js +12 -43
- package/dist/transform/shared.js +4 -4
- package/dist/transform/transform.d.ts +1 -1
- package/dist/transform/transform.d.ts.map +1 -1
- package/dist/transform/transform.js +10 -10
- package/dist/transform/types.d.ts +2 -2
- package/dist/transform/types.d.ts.map +1 -1
- package/dist/transform/worker-pool.d.ts +3 -3
- package/dist/transform/worker-pool.d.ts.map +1 -1
- package/dist/transform/worker-pool.js +2 -2
- package/package.json +13 -4
- package/dist/lib/cache.d.ts +0 -48
- package/dist/lib/cache.d.ts.map +0 -1
- package/dist/lib/cache.js +0 -264
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 j0hanz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -6,28 +6,36 @@
|
|
|
6
6
|
|
|
7
7
|
[](https://lmstudio.ai/install-mcp?name=fetch-url&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBqMGhhbnovZmV0Y2gtdXJsLW1jcEBsYXRlc3QiXX0%3D) [](https://cursor.com/en/install-mcp?name=fetch-url&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsIkBqMGhhbnovZmV0Y2gtdXJsLW1jcEBsYXRlc3QiXX0%3D) [](https://block.github.io/goose/extension?cmd=npx&arg=-y&arg=%40j0hanz%2Ffetch-url-mcp%40latest&id=%40j0hanz%2Ffetch-url-mcp&name=fetch-url&description=fetch-url%20MCP%20server)
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
An MCP server that fetches web pages and converts them to clean, readable Markdown.
|
|
10
10
|
|
|
11
11
|
## Overview
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
This server takes a URL, fetches the page, and strips away everything you don't need — navigation, sidebars, banners, scripts — leaving just the main content as Markdown. It's perfect for feeding into LLMs, giving them the distilled essence of a page without the noise. It also recognizes GitHub, GitLab, Bitbucket, and Gist URLs and rewrites them to fetch the raw content directly.
|
|
14
|
+
|
|
15
|
+
By default it runs over stdio. Pass `--http` if you need a proper HTTP endpoint with auth, rate limiting, TLS, and session support.
|
|
14
16
|
|
|
15
17
|
## Key Features
|
|
16
18
|
|
|
17
|
-
-
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
- `internal://instructions` and `
|
|
21
|
-
- HTTP mode
|
|
19
|
+
- **HTML to Markdown** — Turns any public web page into clean, readable Markdown with metadata like `title`, `url`, `contentSize`, and `truncated`.
|
|
20
|
+
- **Smart URL handling** — Recognizes GitHub, GitLab, Bitbucket, and Gist page URLs and rewrites them to raw-content endpoints before fetching.
|
|
21
|
+
- **Task mode** — Big or slow pages can run as async MCP tasks with progress updates, instead of blocking.
|
|
22
|
+
- **Self-documenting** — Includes an `internal://instructions` resource and a `get-help` prompt so clients know how to use it.
|
|
23
|
+
- **HTTP mode** — Optionally serves over Streamable HTTP with host/origin validation, bearer or OAuth auth, rate limiting, health checks, and TLS.
|
|
24
|
+
|
|
25
|
+
## Web Client
|
|
26
|
+
|
|
27
|
+
A browser-based client is available if you want to use the server without any MCP setup.
|
|
28
|
+
|
|
29
|
+
**[Live app](https://fetch-url-client.vercel.app)** · [Source code](https://github.com/j0hanz/fetch-url)
|
|
22
30
|
|
|
23
31
|
## Requirements
|
|
24
32
|
|
|
25
|
-
- Node.js >=24
|
|
26
|
-
- Docker
|
|
33
|
+
- **Node.js** >= 24
|
|
34
|
+
- **Docker** (optional) — only needed if you want to run the container image
|
|
27
35
|
|
|
28
36
|
## Quick Start
|
|
29
37
|
|
|
30
|
-
|
|
38
|
+
Add this to your MCP client config:
|
|
31
39
|
|
|
32
40
|
```json
|
|
33
41
|
{
|
|
@@ -474,10 +482,9 @@ For more info, see [Kilo Code docs](https://kilocode.ai/docs).
|
|
|
474
482
|
|
|
475
483
|
## Use Cases
|
|
476
484
|
|
|
477
|
-
-
|
|
478
|
-
-
|
|
479
|
-
-
|
|
480
|
-
- Use task mode for large pages or slower sites when the fetch might otherwise be delayed, cancelled, or better handled asynchronously.
|
|
485
|
+
- **Documentation for LLMs** — Grab a docs page, blog post, or reference article as Markdown and pass it straight into a context window.
|
|
486
|
+
- **Repository content** — Hand it a GitHub, GitLab, or Bitbucket URL and it resolves the raw content endpoint. Works with Gists too.
|
|
487
|
+
- **Slow or large pages** — Task mode lets big fetches run in the background while sending progress updates back to the client.
|
|
481
488
|
|
|
482
489
|
## Architecture
|
|
483
490
|
|
|
@@ -488,7 +495,6 @@ For more info, see [Kilo Code docs](https://kilocode.ai/docs).
|
|
|
488
495
|
├─ `GET /health`
|
|
489
496
|
├─ `GET /.well-known/oauth-protected-resource`
|
|
490
497
|
├─ `GET /.well-known/oauth-protected-resource/mcp`
|
|
491
|
-
├─ `GET /mcp/downloads/{namespace}/{hash}`
|
|
492
498
|
└─ `POST|GET|DELETE /mcp`
|
|
493
499
|
|
|
494
500
|
`createMcpServer()`
|
|
@@ -496,15 +502,14 @@ For more info, see [Kilo Code docs](https://kilocode.ai/docs).
|
|
|
496
502
|
├─ registers prompt: `get-help`
|
|
497
503
|
├─ registers resources:
|
|
498
504
|
│ - `internal://instructions`
|
|
499
|
-
|
|
500
|
-
├─ enables capabilities: completions, logging, resources, prompts, tasks
|
|
505
|
+
├─ enables capabilities: logging, resources, prompts, tasks
|
|
501
506
|
└─ installs task handlers, log-level handling, and shutdown cleanup
|
|
502
507
|
|
|
503
508
|
`fetch-url` execution
|
|
504
509
|
├─ validate input with `fetchUrlInputSchema`
|
|
505
510
|
├─ normalize URL and block local/private targets unless allowed
|
|
506
511
|
├─ rewrite supported code-host URLs to raw endpoints when possible
|
|
507
|
-
├─ fetch
|
|
512
|
+
├─ fetch content via the shared pipeline
|
|
508
513
|
├─ transform HTML into Markdown in the transform worker path
|
|
509
514
|
└─ validate `structuredContent` with `fetchUrlOutputSchema`
|
|
510
515
|
```
|
|
@@ -525,14 +530,13 @@ For more info, see [Kilo Code docs](https://kilocode.ai/docs).
|
|
|
525
530
|
|
|
526
531
|
#### `fetch-url`
|
|
527
532
|
|
|
528
|
-
|
|
533
|
+
Takes a URL and returns Markdown. Read-only — no JavaScript execution. Supports running as a background MCP task for large or slow pages.
|
|
529
534
|
|
|
530
|
-
| Parameter
|
|
531
|
-
|
|
|
532
|
-
| `url`
|
|
533
|
-
| `forceRefresh` | `boolean` | no | Bypass cache and fetch fresh content. |
|
|
535
|
+
| Parameter | Type | Required | Description |
|
|
536
|
+
| --------- | -------- | -------- | --------------------------- |
|
|
537
|
+
| `url` | `string` | yes | Target URL. Max 2048 chars. |
|
|
534
538
|
|
|
535
|
-
|
|
539
|
+
You get text content back by default. If output validation passes, the response also includes `structuredContent` with typed fields: `url`, `resolvedUrl`, `finalUrl`, `title`, `metadata`, `markdown`, `fetchedAt`, `contentSize`, and `truncated`. A `true` value for `truncated` means the content hit a server-side size limit.
|
|
536
540
|
|
|
537
541
|
```text
|
|
538
542
|
1. [Client] -- tools/call {name: "fetch-url", arguments} --> [Server]
|
|
@@ -544,27 +548,26 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
544
548
|
|
|
545
549
|
### Resources
|
|
546
550
|
|
|
547
|
-
| Resource | URI
|
|
548
|
-
| ---------------------------- |
|
|
549
|
-
| `fetch-url-mcp-instructions` | `internal://instructions`
|
|
550
|
-
| `fetch-url-mcp-cache-entry` | `internal://cache/{namespace}/{hash}` | `text/markdown` | Read cached markdown generated by previous `fetch-url` calls. |
|
|
551
|
+
| Resource | URI | MIME Type | Description |
|
|
552
|
+
| ---------------------------- | ------------------------- | --------------- | -------------------------------------------- |
|
|
553
|
+
| `fetch-url-mcp-instructions` | `internal://instructions` | `text/markdown` | Guidance for using the Fetch URL MCP server. |
|
|
551
554
|
|
|
552
555
|
### Prompts
|
|
553
556
|
|
|
554
|
-
| Prompt | Arguments | Description
|
|
555
|
-
| ---------- | --------- |
|
|
556
|
-
| `get-help` | none | Return Fetch URL server instructions: workflows,
|
|
557
|
+
| Prompt | Arguments | Description |
|
|
558
|
+
| ---------- | --------- | ------------------------------------------------------------------------------- |
|
|
559
|
+
| `get-help` | none | Return Fetch URL server instructions: workflows, task mode, and error handling. |
|
|
557
560
|
|
|
558
561
|
## MCP Capabilities
|
|
559
562
|
|
|
560
|
-
| Capability | Status | Notes
|
|
561
|
-
| ------------------------------- | --------- |
|
|
562
|
-
| completions | confirmed | Advertised in `createServerCapabilities()
|
|
563
|
-
| logging | confirmed | Advertised in `createServerCapabilities()` and handled through `SetLevelRequestSchema`.
|
|
564
|
-
| resources subscribe/listChanged | confirmed | Advertised in `createServerCapabilities()
|
|
565
|
-
| prompts | confirmed | `get-help` is registered during server startup.
|
|
566
|
-
| tasks | confirmed | Advertised in `createServerCapabilities()` and backed by registered task handlers plus optional tool task support.
|
|
567
|
-
| progress notifications | confirmed | Tool execution reports `notifications/progress` updates during fetch and transform stages.
|
|
563
|
+
| Capability | Status | Notes |
|
|
564
|
+
| ------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------ |
|
|
565
|
+
| completions | confirmed | Advertised in `createServerCapabilities()`. |
|
|
566
|
+
| logging | confirmed | Advertised in `createServerCapabilities()` and handled through `SetLevelRequestSchema`. |
|
|
567
|
+
| resources subscribe/listChanged | confirmed | Advertised in `createServerCapabilities()`. |
|
|
568
|
+
| prompts | confirmed | `get-help` is registered during server startup. |
|
|
569
|
+
| tasks | confirmed | Advertised in `createServerCapabilities()` and backed by registered task handlers plus optional tool task support. |
|
|
570
|
+
| progress notifications | confirmed | Tool execution reports `notifications/progress` updates during fetch and transform stages. |
|
|
568
571
|
|
|
569
572
|
### Tool Annotations
|
|
570
573
|
|
|
@@ -577,59 +580,101 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
577
580
|
|
|
578
581
|
### Structured Output
|
|
579
582
|
|
|
580
|
-
|
|
583
|
+
The tool declares an `outputSchema` and includes `structuredContent` in the response when validation passes. Clients that support structured output get typed data directly; the rest use the text fallback.
|
|
581
584
|
|
|
582
585
|
## Configuration
|
|
583
586
|
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
|
589
|
-
|
|
|
590
|
-
| `
|
|
591
|
-
| `
|
|
592
|
-
| `
|
|
593
|
-
| `
|
|
594
|
-
| `
|
|
595
|
-
| `
|
|
596
|
-
| `
|
|
597
|
-
| `
|
|
598
|
-
| `
|
|
599
|
-
| `
|
|
600
|
-
| `
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
|
605
|
-
|
|
|
606
|
-
| `
|
|
607
|
-
| `
|
|
608
|
-
| `
|
|
609
|
-
| `
|
|
610
|
-
| `
|
|
611
|
-
| `
|
|
612
|
-
| `
|
|
613
|
-
| `
|
|
614
|
-
| `
|
|
615
|
-
| `
|
|
616
|
-
| `
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
|
621
|
-
|
|
|
622
|
-
| `
|
|
623
|
-
| `
|
|
624
|
-
| `
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
|
629
|
-
|
|
|
630
|
-
| `
|
|
631
|
-
| `
|
|
632
|
-
| `
|
|
587
|
+
All configuration is through environment variables. For basic stdio usage, nothing needs to be set.
|
|
588
|
+
|
|
589
|
+
### HTTP Server
|
|
590
|
+
|
|
591
|
+
| Variable | Default | Notes |
|
|
592
|
+
| ------------------------------------- | ----------- | --------------------------------------------------------------------- |
|
|
593
|
+
| `HOST` | `127.0.0.1` | Bind address. Non-loopback bindings also require `ALLOW_REMOTE=true`. |
|
|
594
|
+
| `PORT` | `3000` | Listening port for `--http`. |
|
|
595
|
+
| `ALLOW_REMOTE` | `false` | Must be enabled to bind to a non-loopback interface. |
|
|
596
|
+
| `ALLOWED_HOSTS` | empty | Additional allowed `Host` and `Origin` values. |
|
|
597
|
+
| `SERVER_MAX_CONNECTIONS` | `0` | Optional connection cap. |
|
|
598
|
+
| `SERVER_HEADERS_TIMEOUT_MS` | unset | Optional Node server tuning. |
|
|
599
|
+
| `SERVER_REQUEST_TIMEOUT_MS` | unset | Optional Node server tuning. |
|
|
600
|
+
| `SERVER_KEEP_ALIVE_TIMEOUT_MS` | unset | Optional keep-alive tuning. |
|
|
601
|
+
| `SERVER_KEEP_ALIVE_TIMEOUT_BUFFER_MS` | unset | Optional keep-alive tuning buffer. |
|
|
602
|
+
| `SERVER_MAX_HEADERS_COUNT` | unset | Optional header count limit. |
|
|
603
|
+
| `SERVER_BLOCK_PRIVATE_CONNECTIONS` | `false` | Enables inbound private-network protections. |
|
|
604
|
+
|
|
605
|
+
### Authentication & OAuth
|
|
606
|
+
|
|
607
|
+
| Variable | Default | Notes |
|
|
608
|
+
| ------------------------- | ------- | ----------------------------------------------------------- |
|
|
609
|
+
| `ACCESS_TOKENS` | unset | Comma- or space-separated static bearer tokens. |
|
|
610
|
+
| `API_KEY` | unset | Alternate static token source for header auth. |
|
|
611
|
+
| `OAUTH_ISSUER_URL` | unset | Enables OAuth mode when combined with the other OAuth URLs. |
|
|
612
|
+
| `OAUTH_AUTHORIZATION_URL` | unset | Optional explicit authorization endpoint. |
|
|
613
|
+
| `OAUTH_TOKEN_URL` | unset | Optional explicit token endpoint. |
|
|
614
|
+
| `OAUTH_REVOCATION_URL` | unset | Optional OAuth revocation endpoint. |
|
|
615
|
+
| `OAUTH_REGISTRATION_URL` | unset | Optional OAuth dynamic client registration endpoint. |
|
|
616
|
+
| `OAUTH_INTROSPECTION_URL` | unset | Required for OAuth token introspection. |
|
|
617
|
+
| `OAUTH_REQUIRED_SCOPES` | empty | Required scopes enforced after auth. |
|
|
618
|
+
| `OAUTH_CLIENT_ID` | unset | Optional introspection client ID. |
|
|
619
|
+
| `OAUTH_CLIENT_SECRET` | unset | Optional introspection client secret. |
|
|
620
|
+
|
|
621
|
+
### TLS
|
|
622
|
+
|
|
623
|
+
| Variable | Default | Notes |
|
|
624
|
+
| ---------------------- | ------- | ----------------------------------------------------------- |
|
|
625
|
+
| `SERVER_TLS_KEY_FILE` | unset | Enable HTTPS when set together with `SERVER_TLS_CERT_FILE`. |
|
|
626
|
+
| `SERVER_TLS_CERT_FILE` | unset | TLS certificate path. |
|
|
627
|
+
| `SERVER_TLS_CA_FILE` | unset | Optional custom CA bundle. |
|
|
628
|
+
|
|
629
|
+
### Fetching
|
|
630
|
+
|
|
631
|
+
| Variable | Default | Notes |
|
|
632
|
+
| ------------------- | ------------------------- | -------------------------------------------------- |
|
|
633
|
+
| `ALLOW_LOCAL_FETCH` | `false` | Allows loopback and private-network fetch targets. |
|
|
634
|
+
| `FETCH_TIMEOUT_MS` | `15000` | Network fetch timeout in milliseconds. |
|
|
635
|
+
| `USER_AGENT` | `fetch-url-mcp/<version>` | Override the outbound user agent string. |
|
|
636
|
+
|
|
637
|
+
### Tool Output
|
|
638
|
+
|
|
639
|
+
| Variable | Default | Notes |
|
|
640
|
+
| -------------------------- | ------- | ---------------------------------------------- |
|
|
641
|
+
| `MAX_INLINE_CONTENT_CHARS` | `0` | `0` means no explicit inline truncation limit. |
|
|
642
|
+
|
|
643
|
+
### Tasks
|
|
644
|
+
|
|
645
|
+
| Variable | Default | Notes |
|
|
646
|
+
| ---------------------------- | ------- | ------------------------------------------------------ |
|
|
647
|
+
| `TASKS_MAX_TOTAL` | `5000` | Total task capacity. |
|
|
648
|
+
| `TASKS_MAX_PER_OWNER` | `1000` | Per-owner task cap, clamped to the total cap. |
|
|
649
|
+
| `TASKS_STATUS_NOTIFICATIONS` | `false` | Enables status notifications for tasks. |
|
|
650
|
+
| `TASKS_REQUIRE_INTERCEPTION` | `true` | Requires interception for task-capable tool execution. |
|
|
651
|
+
|
|
652
|
+
### Transform Workers
|
|
653
|
+
|
|
654
|
+
| Variable | Default | Notes |
|
|
655
|
+
| ------------------------------------------ | --------- | ------------------------------------- |
|
|
656
|
+
| `TRANSFORM_CANCEL_ACK_TIMEOUT_MS` | `200` | Cancellation acknowledgement timeout. |
|
|
657
|
+
| `TRANSFORM_WORKER_MODE` | `threads` | Worker execution mode. |
|
|
658
|
+
| `TRANSFORM_WORKER_MAX_OLD_GENERATION_MB` | unset | Optional worker memory limit. |
|
|
659
|
+
| `TRANSFORM_WORKER_MAX_YOUNG_GENERATION_MB` | unset | Optional worker memory limit. |
|
|
660
|
+
| `TRANSFORM_WORKER_CODE_RANGE_MB` | unset | Optional worker memory limit. |
|
|
661
|
+
| `TRANSFORM_WORKER_STACK_MB` | unset | Optional worker stack size. |
|
|
662
|
+
|
|
663
|
+
### Content Cleanup
|
|
664
|
+
|
|
665
|
+
| Variable | Default | Notes |
|
|
666
|
+
| ------------------------------------- | -------------- | ------------------------------------------ |
|
|
667
|
+
| `FETCH_URL_MCP_EXTRA_NOISE_TOKENS` | empty | Extra noise-removal tokens. |
|
|
668
|
+
| `FETCH_URL_MCP_EXTRA_NOISE_SELECTORS` | empty | Extra DOM selectors for noise removal. |
|
|
669
|
+
| `FETCH_URL_MCP_LOCALE` | system default | Locale override for extraction heuristics. |
|
|
670
|
+
| `MARKDOWN_HEADING_KEYWORDS` | built-in list | Override heading keywords used by cleanup. |
|
|
671
|
+
|
|
672
|
+
### Logging
|
|
673
|
+
|
|
674
|
+
| Variable | Default | Notes |
|
|
675
|
+
| ------------ | ------- | ------------------------------------ |
|
|
676
|
+
| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, or `error`. |
|
|
677
|
+
| `LOG_FORMAT` | `text` | Set to `json` for structured logs. |
|
|
633
678
|
|
|
634
679
|
## HTTP Endpoints
|
|
635
680
|
|
|
@@ -641,7 +686,6 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
641
686
|
| `POST` | `/mcp` | yes | Session initialization and JSON-RPC requests. |
|
|
642
687
|
| `GET` | `/mcp` | yes | Session-bound server-to-client stream handling. |
|
|
643
688
|
| `DELETE` | `/mcp` | yes | Session shutdown. |
|
|
644
|
-
| `GET` | `/mcp/downloads/{namespace}/{hash}` | yes | Download route used by HTTP-mode cached fetch results. |
|
|
645
689
|
|
|
646
690
|
## Security
|
|
647
691
|
|
|
@@ -649,7 +693,7 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
649
693
|
| -------------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
|
|
650
694
|
| Host and origin validation | implemented | HTTP requests are rejected unless `Host` and `Origin` match the allowlist built from loopback, the configured host, and `ALLOWED_HOSTS`. |
|
|
651
695
|
| Authentication | implemented | HTTP mode supports static bearer tokens locally or OAuth token introspection; remote bindings require OAuth. |
|
|
652
|
-
| Protocol version checks | implemented | HTTP
|
|
696
|
+
| Protocol version checks | implemented | Session-bound MCP HTTP requests validate `MCP-Protocol-Version` and pin it to the negotiated session version. |
|
|
653
697
|
| Rate limiting | implemented | Requests pass through the HTTP rate limiter before route dispatch. |
|
|
654
698
|
| Outbound SSRF protections | implemented | Local/private IPs, metadata endpoints, and `.local`/`.internal` hosts are blocked unless `ALLOW_LOCAL_FETCH=true`. |
|
|
655
699
|
| TLS | optional | HTTPS is enabled when both TLS key and certificate files are configured. |
|
|
@@ -657,6 +701,24 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
657
701
|
|
|
658
702
|
## Development
|
|
659
703
|
|
|
704
|
+
### Essential Commands
|
|
705
|
+
|
|
706
|
+
| Command | Description |
|
|
707
|
+
| -------------------- | ------------------------------------------------- |
|
|
708
|
+
| `npm run build` | Clean, compile TypeScript, copy assets. |
|
|
709
|
+
| `npm run dev` | Watch mode TypeScript compilation. |
|
|
710
|
+
| `npm run dev:run` | Run the server with `--watch` and `.env` support. |
|
|
711
|
+
| `npm start` | Start the compiled server. |
|
|
712
|
+
| `npm test` | Run the full test suite. |
|
|
713
|
+
| `npm run lint` | Lint with ESLint. |
|
|
714
|
+
| `npm run lint:fix` | Auto-fix lint issues. |
|
|
715
|
+
| `npm run type-check` | Type-check source and tests. |
|
|
716
|
+
| `npm run format` | Format with Prettier. |
|
|
717
|
+
| `npm run inspector` | Build and launch MCP Inspector. |
|
|
718
|
+
|
|
719
|
+
<details>
|
|
720
|
+
<summary><b>All npm scripts</b></summary>
|
|
721
|
+
|
|
660
722
|
| Script | Command |
|
|
661
723
|
| ------------------------ | ------------------------------------------------------------------------------------------------------------------- |
|
|
662
724
|
| `clean` | `node scripts/tasks.mjs clean` |
|
|
@@ -683,17 +745,26 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
683
745
|
| `inspector` | `npm run build && npx -y @modelcontextprotocol/inspector node dist/index.js --stdio` |
|
|
684
746
|
| `prepublishOnly` | `npm run lint && npm run type-check && npm run build` |
|
|
685
747
|
|
|
748
|
+
</details>
|
|
749
|
+
|
|
686
750
|
## Build and Release
|
|
687
751
|
|
|
688
|
-
-
|
|
689
|
-
-
|
|
690
|
-
- `
|
|
752
|
+
- `npm run prepublishOnly` runs lint, type-check, and build as a single release gate.
|
|
753
|
+
- CI workflows are under `.github/workflows/`.
|
|
754
|
+
- `Dockerfile` and `docker-compose.yml` are included for containerized runs.
|
|
755
|
+
- Published on npm as [`@j0hanz/fetch-url-mcp`](https://www.npmjs.com/package/@j0hanz/fetch-url-mcp).
|
|
691
756
|
|
|
692
757
|
## Troubleshooting
|
|
693
758
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
759
|
+
| Symptom | Likely Cause | Fix |
|
|
760
|
+
| --------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------- |
|
|
761
|
+
| Server output mixes with MCP traffic on stdio | Logs going to stdout | Ensure all logging writes to stderr; the server does this by default. |
|
|
762
|
+
| HTTP mode returns `403` | Host/origin mismatch | Add the domain to `ALLOWED_HOSTS` or verify loopback bindings. |
|
|
763
|
+
| HTTP mode returns `401` | Missing or invalid token | Set `ACCESS_TOKENS` or configure OAuth env vars for remote bindings. |
|
|
764
|
+
| Fetch returns private-IP error | SSRF protections blocked the target | Set `ALLOW_LOCAL_FETCH=true` if the target is intentionally local. |
|
|
765
|
+
| `truncated: true` in response | Content exceeded inline limits | Increase `MAX_INLINE_CONTENT_CHARS` or accept truncated output. |
|
|
766
|
+
| Transform timeout or worker crash | Large or complex HTML | Tune `TRANSFORM_WORKER_MAX_OLD_GENERATION_MB` or increase `FETCH_TIMEOUT_MS`. |
|
|
767
|
+
| Client config not working | Wrong config format for the client | Check the matching `<details>` block above — config keys vary by client. |
|
|
697
768
|
|
|
698
769
|
## Credits
|
|
699
770
|
|
|
@@ -708,5 +779,12 @@ The response is returned as MCP text content and, when validation succeeds, as `
|
|
|
708
779
|
|
|
709
780
|
## Contributing and License
|
|
710
781
|
|
|
711
|
-
|
|
712
|
-
|
|
782
|
+
Pull requests welcome. Please make sure these pass before submitting:
|
|
783
|
+
|
|
784
|
+
1. `npm run lint` and `npm run type-check`
|
|
785
|
+
2. `npm test`
|
|
786
|
+
3. `npm run format`
|
|
787
|
+
|
|
788
|
+
## License
|
|
789
|
+
|
|
790
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../src/http/health.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../src/http/health.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAGnD,OAAO,EAAE,KAAK,cAAc,EAAY,MAAM,cAAc,CAAC;AAY7D,wBAAgB,wBAAwB,IAAI,IAAI,CAI/C;AAED,wBAAgB,0BAA0B,IAAI,IAAI,CAEjD;AAqKD,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAKnE;AAiCD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAEpE;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,GAAG,EAAE,cAAc,EACnB,WAAW,EAAE,OAAO,GACnB,OAAO,CAOT"}
|
package/dist/http/health.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { freemem, hostname, totalmem } from 'node:os';
|
|
2
2
|
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
|
3
3
|
import process from 'node:process';
|
|
4
|
-
import { keys as cacheKeys } from '../lib/cache.js';
|
|
5
4
|
import { config, serverVersion } from '../lib/core.js';
|
|
6
5
|
import { getTransformPoolStats } from '../transform/transform.js';
|
|
7
6
|
import { sendJson } from './helpers.js';
|
|
@@ -96,7 +95,6 @@ function buildHealthResponse(store, includeDiagnostics) {
|
|
|
96
95
|
: {}),
|
|
97
96
|
stats: {
|
|
98
97
|
activeSessions: store.size(),
|
|
99
|
-
cacheKeys: cacheKeys().length,
|
|
100
98
|
workerPool: poolStats ?? {
|
|
101
99
|
queueDepth: 0,
|
|
102
100
|
activeWorkers: 0,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"native.d.ts","sourceRoot":"","sources":["../../src/http/native.ts"],"names":[],"mappings":"AAqmCA,wBAAsB,eAAe,IAAI,OAAO,CAAC;IAC/C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,CAAC,CA0DD"}
|
package/dist/http/native.js
CHANGED
|
@@ -8,7 +8,6 @@ import process from 'node:process';
|
|
|
8
8
|
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
|
9
9
|
import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
|
|
10
10
|
import { composeCloseHandlers, config, createSessionStore, createSlotTracker, enableHttpMode, ensureSessionCapacity, logError, logInfo, registerMcpSessionServer, reserveSessionSlot, runWithRequestContext, startSessionCleanupLoop, } from '../lib/core.js';
|
|
11
|
-
import { handleDownload } from '../lib/http.js';
|
|
12
11
|
import { acceptsEventStream, acceptsJsonAndEventStream, isJsonRpcBatchRequest, isMcpMessageBody, isMcpRequestBody, } from '../lib/mcp-interop.js';
|
|
13
12
|
import { applyHttpServerTuning, drainConnectionsOnShutdown, isObject, toError, } from '../lib/utils.js';
|
|
14
13
|
import { createMcpServerForHttpSession } from '../server.js';
|
|
@@ -425,19 +424,6 @@ class McpSessionGateway {
|
|
|
425
424
|
}
|
|
426
425
|
}
|
|
427
426
|
// ---------------------------------------------------------------------------
|
|
428
|
-
// Download route
|
|
429
|
-
// ---------------------------------------------------------------------------
|
|
430
|
-
function checkDownloadRoute(path) {
|
|
431
|
-
const downloadMatch = /^\/mcp\/downloads\/([^/]+)\/([^/]+)$/.exec(path);
|
|
432
|
-
if (!downloadMatch)
|
|
433
|
-
return null;
|
|
434
|
-
const namespace = downloadMatch[1];
|
|
435
|
-
const hash = downloadMatch[2];
|
|
436
|
-
if (!namespace || !hash)
|
|
437
|
-
return null;
|
|
438
|
-
return { namespace, hash };
|
|
439
|
-
}
|
|
440
|
-
// ---------------------------------------------------------------------------
|
|
441
427
|
// HTTP dispatcher
|
|
442
428
|
// ---------------------------------------------------------------------------
|
|
443
429
|
class HttpDispatcher {
|
|
@@ -461,15 +447,6 @@ class HttpDispatcher {
|
|
|
461
447
|
sendHealthRouteResponse(this.store, ctx, true);
|
|
462
448
|
return true;
|
|
463
449
|
}
|
|
464
|
-
tryHandleDownloadRoute(ctx) {
|
|
465
|
-
if (ctx.method !== 'GET')
|
|
466
|
-
return false;
|
|
467
|
-
const download = checkDownloadRoute(ctx.url.pathname);
|
|
468
|
-
if (!download)
|
|
469
|
-
return false;
|
|
470
|
-
handleDownload(ctx.res, download.namespace, download.hash);
|
|
471
|
-
return true;
|
|
472
|
-
}
|
|
473
450
|
tryHandleProtectedResourceMetadataRoute(ctx) {
|
|
474
451
|
if (ctx.method !== 'GET')
|
|
475
452
|
return false;
|
|
@@ -491,8 +468,6 @@ class HttpDispatcher {
|
|
|
491
468
|
if (!auth)
|
|
492
469
|
return;
|
|
493
470
|
const authCtx = { ...ctx, auth };
|
|
494
|
-
if (this.tryHandleDownloadRoute(authCtx))
|
|
495
|
-
return;
|
|
496
471
|
if (isMcpRoute(ctx.url.pathname)) {
|
|
497
472
|
const handled = await this.handleMcpRoutes(authCtx);
|
|
498
473
|
if (handled)
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -75,12 +75,6 @@ interface AppTasksConfig {
|
|
|
75
75
|
emitStatusNotifications: boolean;
|
|
76
76
|
requireInterception: boolean;
|
|
77
77
|
}
|
|
78
|
-
interface AppCacheConfig {
|
|
79
|
-
enabled: boolean;
|
|
80
|
-
ttl: number;
|
|
81
|
-
maxKeys: number;
|
|
82
|
-
maxSizeBytes: number;
|
|
83
|
-
}
|
|
84
78
|
interface AppNoiseRemovalConfig {
|
|
85
79
|
extraTokens: string[];
|
|
86
80
|
extraSelectors: string[];
|
|
@@ -112,7 +106,6 @@ export declare const config: {
|
|
|
112
106
|
timeoutMs: number;
|
|
113
107
|
};
|
|
114
108
|
tasks: AppTasksConfig;
|
|
115
|
-
cache: AppCacheConfig;
|
|
116
109
|
extraction: {
|
|
117
110
|
maxBlockLength: number;
|
|
118
111
|
minParagraphLength: number;
|
package/dist/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAgDA,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AAIxE,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AA2B3D,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AA4OnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAyCD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AAoGD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAQD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAgDA,eAAO,MAAM,aAAa,EAAE,MAA2C,CAAC;AAIxE,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AA2B3D,KAAK,mBAAmB,GAAG,SAAS,GAAG,SAAS,CAAC;AACjD,KAAK,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AA4OnC,UAAU,oBAAoB;IAC5B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAyCD,UAAU,UAAU;IAClB,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,GAAG,GAAG,SAAS,CAAC;IAC3B,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,QAAQ,EAAE,GAAG,GAAG,SAAS,CAAC;IAC1B,aAAa,EAAE,GAAG,GAAG,SAAS,CAAC;IAC/B,eAAe,EAAE,GAAG,GAAG,SAAS,CAAC;IACjC,gBAAgB,EAAE,GAAG,GAAG,SAAS,CAAC;IAClC,WAAW,EAAE,GAAG,CAAC;IACjB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B;AAoGD,UAAU,YAAY;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAQD,UAAU,mBAAmB;IAC3B,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,OAAO,CAAC;IACjC,4BAA4B,EAAE,OAAO,CAAC;IACtC,2BAA2B,EAAE,OAAO,CAAC;CACtC;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,WAAW,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,mBAAmB,CAAC;CAC3B;AAuCD,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAWD,UAAU,kBAAkB;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,mBAAmB,CAAC;IAChC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;CACxD;AAkBD,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAmBD,UAAU,qBAAqB;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AA2BD,UAAU,wBAAwB;IAChC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;IACzB,qBAAqB,EAAE,OAAO,CAAC;IAC/B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAiBD,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8ClB,CAAC;AAEF,wBAAgB,cAAc,IAAI,IAAI,CAErC"}
|
package/dist/lib/config.js
CHANGED
|
@@ -390,14 +390,6 @@ function buildTasksConfig() {
|
|
|
390
390
|
requireInterception: EnvParser.boolean(env['TASKS_REQUIRE_INTERCEPTION'], true, 'TASKS_REQUIRE_INTERCEPTION'),
|
|
391
391
|
};
|
|
392
392
|
}
|
|
393
|
-
function buildCacheConfig() {
|
|
394
|
-
return {
|
|
395
|
-
enabled: EnvParser.boolean(env['CACHE_ENABLED'], true, 'CACHE_ENABLED'),
|
|
396
|
-
ttl: 86400,
|
|
397
|
-
maxKeys: 100,
|
|
398
|
-
maxSizeBytes: 50 * 1024 * 1024,
|
|
399
|
-
};
|
|
400
|
-
}
|
|
401
393
|
function buildNoiseRemovalConfig() {
|
|
402
394
|
return {
|
|
403
395
|
extraTokens: EnvParser.list(env['FETCH_URL_MCP_EXTRA_NOISE_TOKENS']),
|
|
@@ -441,7 +433,6 @@ export const config = {
|
|
|
441
433
|
timeoutMs: DEFAULT_TOOL_TIMEOUT_MS,
|
|
442
434
|
},
|
|
443
435
|
tasks: buildTasksConfig(),
|
|
444
|
-
cache: buildCacheConfig(),
|
|
445
436
|
extraction: {
|
|
446
437
|
maxBlockLength: 5000,
|
|
447
438
|
minParagraphLength: 10,
|
package/dist/lib/core.d.ts
CHANGED
|
@@ -2,8 +2,6 @@ import { type McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
|
2
2
|
import type { SessionEntry } from './session.js';
|
|
3
3
|
import type { SessionStore } from './session.js';
|
|
4
4
|
export { config, enableHttpMode, serverVersion } from './config.js';
|
|
5
|
-
export { createCacheKey, get, getEntryMeta, isEnabled, keys, onCacheUpdate, parseCacheKey, set, } from './cache.js';
|
|
6
|
-
type McpLogLevel = 'debug' | 'info' | 'notice' | 'warning' | 'error' | 'critical' | 'alert' | 'emergency';
|
|
7
5
|
type LogMetadata = Record<string, unknown>;
|
|
8
6
|
interface RequestContext {
|
|
9
7
|
readonly requestId: string;
|
|
@@ -23,7 +21,6 @@ export declare function logInfo(message: string, meta?: LogMetadata): void;
|
|
|
23
21
|
export declare function logDebug(message: string, meta?: LogMetadata): void;
|
|
24
22
|
export declare function logWarn(message: string, meta?: LogMetadata): void;
|
|
25
23
|
export declare function logError(message: string, error?: Error | LogMetadata): void;
|
|
26
|
-
export declare function getMcpLogLevel(sessionId?: string): McpLogLevel;
|
|
27
24
|
export declare function setLogLevel(level: string, sessionId?: string): void;
|
|
28
25
|
export declare function redactUrl(rawUrl: string): string;
|
|
29
26
|
export type { SessionEntry, SessionStore } from './session.js';
|
package/dist/lib/core.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAOjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/lib/core.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAGzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAOjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAYpE,KAAK,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC3C,UAAU,cAAc;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAkBD,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAKpD;AACD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,SAAS,GAChB,IAAI,CAGN;AACD,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAIlE;AACD,wBAAgB,kCAAkC,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAM1E;AACD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,SAAS,GAChB,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,CAAC,GACV,CAAC,CAEH;AAID,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AACD,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAEjD;AACD,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AAyPD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AACD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAElE;AACD,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,IAAI,CAEjE;AAcD,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,WAAW,GAAG,IAAI,CAI3E;AACD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAUnE;AACD,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAQhD;AACD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAmJtB,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE;IACR,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,eAAe,CAOjB"}
|