clearfetch 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -2
- package/bin/clearfetch-mcp.js +8 -0
- package/dist/client.d.ts +73 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +66 -0
- package/dist/client.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +262 -0
- package/dist/index.js.map +1 -0
- package/dist/tools.d.ts +50 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +58 -0
- package/dist/tools.js.map +1 -0
- package/package.json +47 -4
package/README.md
CHANGED
|
@@ -1,3 +1,125 @@
|
|
|
1
|
-
#
|
|
1
|
+
# clearfetch
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **The Model Context Protocol (MCP) Server for ClearFetch (`clearfetch.dev`)**
|
|
4
|
+
> Where Jina Reader and Firecrawl fail with `403 Forbidden` on Cloudflare Turnstile, DataDome, PerimeterX, or TLS/JA4 fingerprinting, ClearFetch guarantees clean, token-optimized Markdown and structured JSON extraction for AI agents.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## ⚡ Quick Start
|
|
9
|
+
|
|
10
|
+
Run directly via `npx` with zero installation:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
npx -y clearfetch
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
### Required Environment Variables
|
|
17
|
+
|
|
18
|
+
- `CLEARFETCH_API_KEY`: Your live API key from [ClearFetch Dashboard](https://clearfetch.dev/dashboard) (starts with `cf_live_...`).
|
|
19
|
+
- `CLEARFETCH_API_URL`: (Optional) Defaults to `https://api.clearfetch.dev`.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## 🛠 Available Agent Tools
|
|
24
|
+
|
|
25
|
+
| Tool | Description | Key Parameters |
|
|
26
|
+
| :--- | :--- | :--- |
|
|
27
|
+
| `clearfetch_read` | Reads any URL (including Cloudflare/DataDome protected pages) as clean, token-optimized Markdown. | `url` (required), `stealth` ("auto" \| "force"), `max_age` (cache TTL in sec), `only_main_content` (bool) |
|
|
28
|
+
| `clearfetch_extract` | Extracts typed JSON matching a strict schema from any webpage. | `url` (required), `json_schema` (required object), `prompt` (optional instruction) |
|
|
29
|
+
| `clearfetch_search` | Searches the web and returns full clean Markdown content of top results in one step. | `query` (required), `limit` (1-10, default 5), `scrape_results` (bool) |
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 💻 Client Configuration Examples
|
|
34
|
+
|
|
35
|
+
### 1. n8n (AI Agent Workflows)
|
|
36
|
+
|
|
37
|
+
In your n8n workflow with the **AI Agent** node:
|
|
38
|
+
1. Add the **Model Context Protocol (MCP) Tool** node connected to your AI Agent.
|
|
39
|
+
2. Configure Execution:
|
|
40
|
+
- **Command:** `npx`
|
|
41
|
+
- **Arguments:** `-y, clearfetch`
|
|
42
|
+
- **Environment Variables:**
|
|
43
|
+
- Key: `CLEARFETCH_API_KEY`, Value: `cf_live_your_api_key_here`
|
|
44
|
+
3. The AI Agent in n8n can now autonomously browse, unblock Turnstile-protected pages, and extract structured data!
|
|
45
|
+
|
|
46
|
+
*(Alternatively in n8n, you can use the standard **HTTP Request node** directly against `https://api.clearfetch.dev/v1/read`).*
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
### 2. Claude Desktop (`claude_desktop_config.json`)
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"mcpServers": {
|
|
55
|
+
"clearfetch": {
|
|
56
|
+
"command": "npx",
|
|
57
|
+
"args": ["-y", "clearfetch"],
|
|
58
|
+
"env": {
|
|
59
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
### 3. Cursor IDE (`.cursor/mcp.json`)
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{
|
|
72
|
+
"mcpServers": {
|
|
73
|
+
"clearfetch": {
|
|
74
|
+
"command": "npx",
|
|
75
|
+
"args": ["-y", "clearfetch"],
|
|
76
|
+
"env": {
|
|
77
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
### 4. Windsurf / Cline / Roo-Code
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{
|
|
90
|
+
"clearfetch": {
|
|
91
|
+
"command": "npx",
|
|
92
|
+
"args": ["-y", "clearfetch"],
|
|
93
|
+
"env": {
|
|
94
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
95
|
+
},
|
|
96
|
+
"disabled": false,
|
|
97
|
+
"autoApprove": [
|
|
98
|
+
"clearfetch_read",
|
|
99
|
+
"clearfetch_extract",
|
|
100
|
+
"clearfetch_search"
|
|
101
|
+
]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 📦 Building from Source
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
cd packages/mcp-server
|
|
112
|
+
npm install
|
|
113
|
+
npm run build
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Verify stdio connectivity:
|
|
117
|
+
```bash
|
|
118
|
+
node ./bin/clearfetch-mcp.js
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## 🛡 License
|
|
124
|
+
|
|
125
|
+
MIT License. Maintained by the ClearFetch Infrastructure Team (`clearfetch.dev`).
|
package/dist/client.d.ts
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export interface ReadUrlParams {
|
|
2
|
+
url: string;
|
|
3
|
+
stealth?: "auto" | "force";
|
|
4
|
+
max_age?: number;
|
|
5
|
+
include_links?: boolean;
|
|
6
|
+
only_main_content?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface ReadUrlResponse {
|
|
9
|
+
success: boolean;
|
|
10
|
+
url: string;
|
|
11
|
+
title: string;
|
|
12
|
+
description: string;
|
|
13
|
+
markdown: string;
|
|
14
|
+
links: string[];
|
|
15
|
+
metadata: {
|
|
16
|
+
status_code: number;
|
|
17
|
+
cached: boolean;
|
|
18
|
+
stealth_used: boolean;
|
|
19
|
+
estimated_tokens: number;
|
|
20
|
+
latency_ms: number;
|
|
21
|
+
credits_used: number;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export interface ExtractDataParams {
|
|
25
|
+
url: string;
|
|
26
|
+
json_schema: Record<string, unknown>;
|
|
27
|
+
prompt?: string;
|
|
28
|
+
stealth?: "auto" | "force";
|
|
29
|
+
}
|
|
30
|
+
export interface ExtractDataResponse {
|
|
31
|
+
success: boolean;
|
|
32
|
+
url: string;
|
|
33
|
+
data: Record<string, unknown>;
|
|
34
|
+
metadata: {
|
|
35
|
+
status_code: number;
|
|
36
|
+
cached: boolean;
|
|
37
|
+
stealth_used: boolean;
|
|
38
|
+
estimated_tokens: number;
|
|
39
|
+
latency_ms: number;
|
|
40
|
+
credits_used: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export interface SearchAndReadParams {
|
|
44
|
+
query: string;
|
|
45
|
+
limit?: number;
|
|
46
|
+
scrape_results?: boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface SearchResultItem {
|
|
49
|
+
title: string;
|
|
50
|
+
url: string;
|
|
51
|
+
snippet: string;
|
|
52
|
+
markdown?: string;
|
|
53
|
+
estimated_tokens?: number;
|
|
54
|
+
}
|
|
55
|
+
export interface SearchAndReadResponse {
|
|
56
|
+
success: boolean;
|
|
57
|
+
query: string;
|
|
58
|
+
count: number;
|
|
59
|
+
results: SearchResultItem[];
|
|
60
|
+
total_estimated_tokens: number;
|
|
61
|
+
credits_used: number;
|
|
62
|
+
}
|
|
63
|
+
export declare class ClearFetchApiClient {
|
|
64
|
+
private baseUrl;
|
|
65
|
+
private apiKey;
|
|
66
|
+
constructor(apiKey?: string, baseUrl?: string);
|
|
67
|
+
private request;
|
|
68
|
+
readUrl(params: ReadUrlParams): Promise<ReadUrlResponse>;
|
|
69
|
+
extractData(params: ExtractDataParams): Promise<ExtractDataResponse>;
|
|
70
|
+
searchAndRead(params: SearchAndReadParams): Promise<SearchAndReadResponse>;
|
|
71
|
+
}
|
|
72
|
+
export { ClearFetchApiClient as UnblockedApiClient };
|
|
73
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;YAW/B,OAAO;IAgCf,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAUxD,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IASpE,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAOjF;AAGD,OAAO,EAAE,mBAAmB,IAAI,kBAAkB,EAAE,CAAC"}
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export class ClearFetchApiClient {
|
|
2
|
+
baseUrl;
|
|
3
|
+
apiKey;
|
|
4
|
+
constructor(apiKey, baseUrl) {
|
|
5
|
+
this.apiKey = apiKey || process.env.CLEARFETCH_API_KEY || process.env.UNBLOCKED_API_KEY || "";
|
|
6
|
+
this.baseUrl = (baseUrl || process.env.CLEARFETCH_API_URL || process.env.UNBLOCKED_API_URL || "https://api.clearfetch.dev").replace(/\/$/, "");
|
|
7
|
+
if (!this.apiKey) {
|
|
8
|
+
console.error("[WARN] CLEARFETCH_API_KEY environment variable is not set. MCP tools will fail authorization if not provided.");
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
async request(endpoint, body) {
|
|
12
|
+
const url = `${this.baseUrl}${endpoint}`;
|
|
13
|
+
const headers = {
|
|
14
|
+
"Content-Type": "application/json",
|
|
15
|
+
"User-Agent": "ClearFetch-MCP-Server/1.0.1",
|
|
16
|
+
};
|
|
17
|
+
if (this.apiKey) {
|
|
18
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
19
|
+
}
|
|
20
|
+
const response = await fetch(url, {
|
|
21
|
+
method: "POST",
|
|
22
|
+
headers,
|
|
23
|
+
body: JSON.stringify(body),
|
|
24
|
+
});
|
|
25
|
+
if (!response.ok) {
|
|
26
|
+
const errorText = await response.text();
|
|
27
|
+
let errorDetail = errorText;
|
|
28
|
+
try {
|
|
29
|
+
const errorJson = JSON.parse(errorText);
|
|
30
|
+
errorDetail = errorJson.detail || errorText;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// ignore json parse error
|
|
34
|
+
}
|
|
35
|
+
throw new Error(`ClearFetch API Error (${response.status}): ${errorDetail}`);
|
|
36
|
+
}
|
|
37
|
+
return (await response.json());
|
|
38
|
+
}
|
|
39
|
+
async readUrl(params) {
|
|
40
|
+
return this.request("/v1/read", {
|
|
41
|
+
url: params.url,
|
|
42
|
+
stealth: params.stealth || "auto",
|
|
43
|
+
max_age: params.max_age ?? 3600,
|
|
44
|
+
include_links: params.include_links ?? true,
|
|
45
|
+
only_main_content: params.only_main_content ?? true,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
async extractData(params) {
|
|
49
|
+
return this.request("/v1/extract", {
|
|
50
|
+
url: params.url,
|
|
51
|
+
json_schema: params.json_schema,
|
|
52
|
+
prompt: params.prompt,
|
|
53
|
+
stealth: params.stealth || "auto",
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
async searchAndRead(params) {
|
|
57
|
+
return this.request("/v1/search", {
|
|
58
|
+
query: params.query,
|
|
59
|
+
limit: params.limit ?? 5,
|
|
60
|
+
scrape_results: params.scrape_results ?? true,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// Backward-compatibility alias
|
|
65
|
+
export { ClearFetchApiClient as UnblockedApiClient };
|
|
66
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAqEA,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAAS;IAChB,MAAM,CAAS;IAEvB,YAAY,MAAe,EAAE,OAAgB;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;QAC9F,IAAI,CAAC,OAAO,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,4BAA4B,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAE/I,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,+GAA+G,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAI,QAAgB,EAAE,IAAa;QACtD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;QACzC,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;YAClC,YAAY,EAAE,6BAA6B;SAC5C,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,WAAW,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxC,WAAW,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,MAAM,WAAW,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAqB;QACjC,OAAO,IAAI,CAAC,OAAO,CAAkB,UAAU,EAAE;YAC/C,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,IAAI;YAC3C,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;SACpD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAyB;QACzC,OAAO,IAAI,CAAC,OAAO,CAAsB,aAAa,EAAE;YACtD,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAA2B;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAwB,YAAY,EAAE;YACvD,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;SAC9C,CAAC,CAAC;IACL,CAAC;CACF;AAED,+BAA+B;AAC/B,OAAO,EAAE,mBAAmB,IAAI,kBAAkB,EAAE,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAqJnE,wBAAgB,YAAY,IAAI,MAAM,CAwIrC;AAED,wBAAsB,cAAc,kBAKnC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
2
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
|
+
import { CallToolRequestSchema, ListToolsRequestSchema, ListPromptsRequestSchema, GetPromptRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
4
|
+
import { ClearFetchApiClient } from "./client.js";
|
|
5
|
+
import { ReadUrlSchema, ExtractDataSchema, SearchAndReadSchema } from "./tools.js";
|
|
6
|
+
import dotenv from "dotenv";
|
|
7
|
+
dotenv.config();
|
|
8
|
+
const client = new ClearFetchApiClient();
|
|
9
|
+
const TOOLS = [
|
|
10
|
+
{
|
|
11
|
+
name: "clearfetch_read",
|
|
12
|
+
description: "Read any URL as clean, token-optimized Markdown. Reliably bypasses Cloudflare Turnstile, DataDome, PerimeterX, and anti-bot systems where other scrapers fail with 403 Forbidden.",
|
|
13
|
+
inputSchema: {
|
|
14
|
+
type: "object",
|
|
15
|
+
properties: {
|
|
16
|
+
url: {
|
|
17
|
+
type: "string",
|
|
18
|
+
description: "The full target URL to read (e.g. https://www.bloomberg.com/news/...)",
|
|
19
|
+
},
|
|
20
|
+
stealth: {
|
|
21
|
+
type: "string",
|
|
22
|
+
enum: ["auto", "force"],
|
|
23
|
+
default: "auto",
|
|
24
|
+
description: "'auto' attempts fast TLS spoofing first and auto-escalates on challenge detection; 'force' routes through full residential browser solver immediately.",
|
|
25
|
+
},
|
|
26
|
+
max_age: {
|
|
27
|
+
type: "integer",
|
|
28
|
+
default: 3600,
|
|
29
|
+
description: "Cache TTL in seconds. 0 forces live fresh fetch.",
|
|
30
|
+
},
|
|
31
|
+
include_links: {
|
|
32
|
+
type: "boolean",
|
|
33
|
+
default: true,
|
|
34
|
+
description: "Whether to return extracted outgoing links.",
|
|
35
|
+
},
|
|
36
|
+
only_main_content: {
|
|
37
|
+
type: "boolean",
|
|
38
|
+
default: true,
|
|
39
|
+
description: "Whether to strip navigation, ads, and footers.",
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
required: ["url"],
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
name: "clearfetch_extract",
|
|
47
|
+
description: "Extract strongly-typed JSON matching a strict schema from any webpage, even behind heavy anti-bot protections.",
|
|
48
|
+
inputSchema: {
|
|
49
|
+
type: "object",
|
|
50
|
+
properties: {
|
|
51
|
+
url: {
|
|
52
|
+
type: "string",
|
|
53
|
+
description: "Target webpage URL to extract data from",
|
|
54
|
+
},
|
|
55
|
+
json_schema: {
|
|
56
|
+
type: "object",
|
|
57
|
+
description: "Strict JSON Schema describing the target data fields",
|
|
58
|
+
},
|
|
59
|
+
prompt: {
|
|
60
|
+
type: "string",
|
|
61
|
+
description: "Optional custom instruction or focus for extraction",
|
|
62
|
+
},
|
|
63
|
+
stealth: {
|
|
64
|
+
type: "string",
|
|
65
|
+
enum: ["auto", "force"],
|
|
66
|
+
default: "auto",
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
required: ["url", "json_schema"],
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
name: "clearfetch_search",
|
|
74
|
+
description: "Search the live web and return clean Markdown content bundles of top results in a single step for AI agent context / RAG.",
|
|
75
|
+
inputSchema: {
|
|
76
|
+
type: "object",
|
|
77
|
+
properties: {
|
|
78
|
+
query: {
|
|
79
|
+
type: "string",
|
|
80
|
+
description: "Search query string",
|
|
81
|
+
},
|
|
82
|
+
limit: {
|
|
83
|
+
type: "integer",
|
|
84
|
+
default: 5,
|
|
85
|
+
minimum: 1,
|
|
86
|
+
maximum: 10,
|
|
87
|
+
description: "Number of search results to return (1-10)",
|
|
88
|
+
},
|
|
89
|
+
scrape_results: {
|
|
90
|
+
type: "boolean",
|
|
91
|
+
default: true,
|
|
92
|
+
description: "Whether to fetch full clean Markdown for each result",
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
required: ["query"],
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
];
|
|
99
|
+
const PROMPTS = [
|
|
100
|
+
{
|
|
101
|
+
name: "read_and_summarize",
|
|
102
|
+
description: "Read any webpage or article behind Cloudflare/Turnstile and generate a concise summary",
|
|
103
|
+
arguments: [
|
|
104
|
+
{
|
|
105
|
+
name: "url",
|
|
106
|
+
description: "Target webpage URL to read (e.g. https://news.ycombinator.com)",
|
|
107
|
+
required: true,
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
name: "extract_pricing_table",
|
|
113
|
+
description: "Extract tiers, prices, and features from a SaaS pricing page into clean JSON",
|
|
114
|
+
arguments: [
|
|
115
|
+
{
|
|
116
|
+
name: "url",
|
|
117
|
+
description: "Pricing page URL (e.g. https://clearfetch.dev/pricing)",
|
|
118
|
+
required: true,
|
|
119
|
+
},
|
|
120
|
+
],
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
name: "web_deep_research",
|
|
124
|
+
description: "Search the live web and read top results to research any topic or query",
|
|
125
|
+
arguments: [
|
|
126
|
+
{
|
|
127
|
+
name: "query",
|
|
128
|
+
description: "Research topic or query (e.g. Model Context Protocol best practices 2026)",
|
|
129
|
+
required: true,
|
|
130
|
+
},
|
|
131
|
+
],
|
|
132
|
+
},
|
|
133
|
+
];
|
|
134
|
+
export function createServer() {
|
|
135
|
+
const server = new Server({
|
|
136
|
+
name: "clearfetch",
|
|
137
|
+
version: "1.0.1",
|
|
138
|
+
}, {
|
|
139
|
+
capabilities: {
|
|
140
|
+
tools: {},
|
|
141
|
+
prompts: {},
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
145
|
+
return { tools: TOOLS };
|
|
146
|
+
});
|
|
147
|
+
server.setRequestHandler(ListPromptsRequestSchema, async () => {
|
|
148
|
+
return { prompts: PROMPTS };
|
|
149
|
+
});
|
|
150
|
+
server.setRequestHandler(GetPromptRequestSchema, async (request) => {
|
|
151
|
+
const { name, arguments: args } = request.params;
|
|
152
|
+
if (name === "read_and_summarize") {
|
|
153
|
+
const url = args?.url || "https://example.com";
|
|
154
|
+
return {
|
|
155
|
+
messages: [
|
|
156
|
+
{
|
|
157
|
+
role: "user",
|
|
158
|
+
content: {
|
|
159
|
+
type: "text",
|
|
160
|
+
text: `Please use the clearfetch_read tool to fetch the content from ${url} and provide a well-structured summary with key takeaways.`,
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
],
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
if (name === "extract_pricing_table") {
|
|
167
|
+
const url = args?.url || "https://clearfetch.dev/pricing";
|
|
168
|
+
return {
|
|
169
|
+
messages: [
|
|
170
|
+
{
|
|
171
|
+
role: "user",
|
|
172
|
+
content: {
|
|
173
|
+
type: "text",
|
|
174
|
+
text: `Please use the clearfetch_extract tool on ${url} to extract all pricing plans, monthly/annual costs, and included limits/features into structured JSON.`,
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
],
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
if (name === "web_deep_research") {
|
|
181
|
+
const query = args?.query || "Model Context Protocol";
|
|
182
|
+
return {
|
|
183
|
+
messages: [
|
|
184
|
+
{
|
|
185
|
+
role: "user",
|
|
186
|
+
content: {
|
|
187
|
+
type: "text",
|
|
188
|
+
text: `Please use the clearfetch_search tool with the query "${query}" and write a detailed synthesis report covering the top findings.`,
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
],
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
throw new Error(`Unknown prompt requested: ${name}`);
|
|
195
|
+
});
|
|
196
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
197
|
+
const { name, arguments: args } = request.params;
|
|
198
|
+
try {
|
|
199
|
+
if (name === "clearfetch_read" || name === "read_url") {
|
|
200
|
+
const validated = ReadUrlSchema.parse(args);
|
|
201
|
+
const res = await client.readUrl(validated);
|
|
202
|
+
return {
|
|
203
|
+
content: [
|
|
204
|
+
{
|
|
205
|
+
type: "text",
|
|
206
|
+
text: `# ${res.title}\n\n${res.markdown}\n\n---\n*Metadata: status=${res.metadata.status_code}, cached=${res.metadata.cached}, stealth=${res.metadata.stealth_used}, tokens=${res.metadata.estimated_tokens}, latency=${res.metadata.latency_ms}ms*`,
|
|
207
|
+
},
|
|
208
|
+
],
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
if (name === "clearfetch_extract" || name === "extract_structured_data") {
|
|
212
|
+
const validated = ExtractDataSchema.parse(args);
|
|
213
|
+
const res = await client.extractData(validated);
|
|
214
|
+
return {
|
|
215
|
+
content: [
|
|
216
|
+
{
|
|
217
|
+
type: "text",
|
|
218
|
+
text: JSON.stringify(res.data, null, 2),
|
|
219
|
+
},
|
|
220
|
+
],
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
if (name === "clearfetch_search" || name === "search_and_read") {
|
|
224
|
+
const validated = SearchAndReadSchema.parse(args);
|
|
225
|
+
const res = await client.searchAndRead(validated);
|
|
226
|
+
const formattedResults = res.results
|
|
227
|
+
.map((item, index) => {
|
|
228
|
+
return `### [${index + 1}] ${item.title}\nURL: ${item.url}\n${item.markdown ? `\nContent:\n${item.markdown}\n` : `\nSnippet: ${item.snippet}\n`}`;
|
|
229
|
+
})
|
|
230
|
+
.join("\n---\n\n");
|
|
231
|
+
return {
|
|
232
|
+
content: [
|
|
233
|
+
{
|
|
234
|
+
type: "text",
|
|
235
|
+
text: `# Search Results for: "${res.query}"\n\n${formattedResults}\n\n---\n*Total tokens: ${res.total_estimated_tokens}, Results count: ${res.count}*`,
|
|
236
|
+
},
|
|
237
|
+
],
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
throw new Error(`Unknown tool requested: ${name}`);
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
return {
|
|
244
|
+
isError: true,
|
|
245
|
+
content: [
|
|
246
|
+
{
|
|
247
|
+
type: "text",
|
|
248
|
+
text: `Error executing ${name}: ${error.message || String(error)}`,
|
|
249
|
+
},
|
|
250
|
+
],
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
return server;
|
|
255
|
+
}
|
|
256
|
+
export async function runStdioServer() {
|
|
257
|
+
const server = createServer();
|
|
258
|
+
const transport = new StdioServerTransport();
|
|
259
|
+
await server.connect(transport);
|
|
260
|
+
console.error("ClearFetch MCP Server running on stdio transport.");
|
|
261
|
+
}
|
|
262
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,EACtB,wBAAwB,EACxB,sBAAsB,GAGvB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;AAEzC,MAAM,KAAK,GAAW;IACpB;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EACT,mLAAmL;QACrL,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uEAAuE;iBACrF;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;oBACf,WAAW,EACT,wJAAwJ;iBAC3J;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,kDAAkD;iBAChE;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,6CAA6C;iBAC3D;gBACD,iBAAiB,EAAE;oBACjB,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,gDAAgD;iBAC9D;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EACT,gHAAgH;QAClH,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,yCAAyC;iBACvD;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,sDAAsD;iBACpE;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qDAAqD;iBACnE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;iBAChB;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC;SACjC;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,WAAW,EACT,2HAA2H;QAC7H,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qBAAqB;iBACnC;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;oBACX,WAAW,EAAE,2CAA2C;iBACzD;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,sDAAsD;iBACpE;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;KACF;CACF,CAAC;AAEF,MAAM,OAAO,GAAa;IACxB;QACE,IAAI,EAAE,oBAAoB;QAC1B,WAAW,EAAE,wFAAwF;QACrG,SAAS,EAAE;YACT;gBACE,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,gEAAgE;gBAC7E,QAAQ,EAAE,IAAI;aACf;SACF;KACF;IACD;QACE,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,8EAA8E;QAC3F,SAAS,EAAE;YACT;gBACE,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,wDAAwD;gBACrE,QAAQ,EAAE,IAAI;aACf;SACF;KACF;IACD;QACE,IAAI,EAAE,mBAAmB;QACzB,WAAW,EAAE,yEAAyE;QACtF,SAAS,EAAE;YACT;gBACE,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,2EAA2E;gBACxF,QAAQ,EAAE,IAAI;aACf;SACF;KACF;CACF,CAAC;AAEF,MAAM,UAAU,YAAY;IAC1B,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;QACE,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,OAAO,EAAE,EAAE;SACZ;KACF,CACF,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QAC5D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACjE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,EAAE,GAAG,IAAI,qBAAqB,CAAC;YAC/C,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,iEAAiE,GAAG,4DAA4D;yBACvI;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,KAAK,uBAAuB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,IAAI,EAAE,GAAG,IAAI,gCAAgC,CAAC;YAC1D,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,6CAA6C,GAAG,yGAAyG;yBAChK;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,wBAAwB,CAAC;YACtD,OAAO;gBACL,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,yDAAyD,KAAK,oEAAoE;yBACzI;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAEjD,IAAI,CAAC;YACH,IAAI,IAAI,KAAK,iBAAiB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;gBACtD,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC5C,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,KAAK,GAAG,CAAC,KAAK,OAAO,GAAG,CAAC,QAAQ,8BAA8B,GAAG,CAAC,QAAQ,CAAC,WAAW,YAAY,GAAG,CAAC,QAAQ,CAAC,MAAM,aAAa,GAAG,CAAC,QAAQ,CAAC,YAAY,YAAY,GAAG,CAAC,QAAQ,CAAC,gBAAgB,aAAa,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK;yBACrP;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,oBAAoB,IAAI,IAAI,KAAK,yBAAyB,EAAE,CAAC;gBACxE,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAChD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;yBACxC;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,mBAAmB,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBAC/D,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBAElD,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO;qBACjC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBACnB,OAAO,QAAQ,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,KACvD,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,OAAO,IAC/E,EAAE,CAAC;gBACL,CAAC,CAAC;qBACD,IAAI,CAAC,WAAW,CAAC,CAAC;gBAErB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,0BAA0B,GAAG,CAAC,KAAK,QAAQ,gBAAgB,2BAA2B,GAAG,CAAC,sBAAsB,oBAAoB,GAAG,CAAC,KAAK,GAAG;yBACvJ;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,mBAAmB,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE;qBACnE;iBACF;aACF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;AACrE,CAAC"}
|
package/dist/tools.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const ReadUrlSchema: z.ZodObject<{
|
|
3
|
+
url: z.ZodString;
|
|
4
|
+
stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
|
|
5
|
+
max_age: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
6
|
+
include_links: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
7
|
+
only_main_content: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
8
|
+
}, "strip", z.ZodTypeAny, {
|
|
9
|
+
url: string;
|
|
10
|
+
stealth: "auto" | "force";
|
|
11
|
+
max_age: number;
|
|
12
|
+
include_links: boolean;
|
|
13
|
+
only_main_content: boolean;
|
|
14
|
+
}, {
|
|
15
|
+
url: string;
|
|
16
|
+
stealth?: "auto" | "force" | undefined;
|
|
17
|
+
max_age?: number | undefined;
|
|
18
|
+
include_links?: boolean | undefined;
|
|
19
|
+
only_main_content?: boolean | undefined;
|
|
20
|
+
}>;
|
|
21
|
+
export declare const ExtractDataSchema: z.ZodObject<{
|
|
22
|
+
url: z.ZodString;
|
|
23
|
+
json_schema: z.ZodRecord<z.ZodString, z.ZodAny>;
|
|
24
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
25
|
+
stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
|
|
26
|
+
}, "strip", z.ZodTypeAny, {
|
|
27
|
+
url: string;
|
|
28
|
+
stealth: "auto" | "force";
|
|
29
|
+
json_schema: Record<string, any>;
|
|
30
|
+
prompt?: string | undefined;
|
|
31
|
+
}, {
|
|
32
|
+
url: string;
|
|
33
|
+
json_schema: Record<string, any>;
|
|
34
|
+
stealth?: "auto" | "force" | undefined;
|
|
35
|
+
prompt?: string | undefined;
|
|
36
|
+
}>;
|
|
37
|
+
export declare const SearchAndReadSchema: z.ZodObject<{
|
|
38
|
+
query: z.ZodString;
|
|
39
|
+
limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
40
|
+
scrape_results: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
41
|
+
}, "strip", z.ZodTypeAny, {
|
|
42
|
+
query: string;
|
|
43
|
+
limit: number;
|
|
44
|
+
scrape_results: boolean;
|
|
45
|
+
}, {
|
|
46
|
+
query: string;
|
|
47
|
+
limit?: number | undefined;
|
|
48
|
+
scrape_results?: boolean | undefined;
|
|
49
|
+
}>;
|
|
50
|
+
//# sourceMappingURL=tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;EA0BxB,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;EAc5B,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;EAe9B,CAAC"}
|
package/dist/tools.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const ReadUrlSchema = z.object({
|
|
3
|
+
url: z.string().url().describe("The full web URL to scrape and read into clean Markdown"),
|
|
4
|
+
stealth: z
|
|
5
|
+
.enum(["auto", "force"])
|
|
6
|
+
.optional()
|
|
7
|
+
.default("auto")
|
|
8
|
+
.describe("'auto' tries fast TLS spoofing first and auto-escalates on 403/Turnstile; 'force' routes through full residential proxy + Turnstile solver immediately."),
|
|
9
|
+
max_age: z
|
|
10
|
+
.number()
|
|
11
|
+
.int()
|
|
12
|
+
.nonnegative()
|
|
13
|
+
.optional()
|
|
14
|
+
.default(3600)
|
|
15
|
+
.describe("Redis cache TTL in seconds. Use 0 to force a real-time fresh bypass fetch."),
|
|
16
|
+
include_links: z
|
|
17
|
+
.boolean()
|
|
18
|
+
.optional()
|
|
19
|
+
.default(true)
|
|
20
|
+
.describe("Whether to include extracted outgoing links in the response"),
|
|
21
|
+
only_main_content: z
|
|
22
|
+
.boolean()
|
|
23
|
+
.optional()
|
|
24
|
+
.default(true)
|
|
25
|
+
.describe("Whether to strip navigational headers, footers, and ad noise using Trafilatura"),
|
|
26
|
+
});
|
|
27
|
+
export const ExtractDataSchema = z.object({
|
|
28
|
+
url: z.string().url().describe("Target webpage URL to extract data from"),
|
|
29
|
+
json_schema: z
|
|
30
|
+
.record(z.any())
|
|
31
|
+
.describe("JSON Schema definition defining the desired typed structure"),
|
|
32
|
+
prompt: z
|
|
33
|
+
.string()
|
|
34
|
+
.optional()
|
|
35
|
+
.describe("Optional extraction prompt or instructions for the LLM"),
|
|
36
|
+
stealth: z
|
|
37
|
+
.enum(["auto", "force"])
|
|
38
|
+
.optional()
|
|
39
|
+
.default("auto")
|
|
40
|
+
.describe("Anti-bot bypass level"),
|
|
41
|
+
});
|
|
42
|
+
export const SearchAndReadSchema = z.object({
|
|
43
|
+
query: z.string().min(1).describe("Search query to execute"),
|
|
44
|
+
limit: z
|
|
45
|
+
.number()
|
|
46
|
+
.int()
|
|
47
|
+
.min(1)
|
|
48
|
+
.max(10)
|
|
49
|
+
.optional()
|
|
50
|
+
.default(5)
|
|
51
|
+
.describe("Number of top results to return (1-10)"),
|
|
52
|
+
scrape_results: z
|
|
53
|
+
.boolean()
|
|
54
|
+
.optional()
|
|
55
|
+
.default(true)
|
|
56
|
+
.describe("Whether to concurrently fetch and convert each search result into clean Markdown"),
|
|
57
|
+
});
|
|
58
|
+
//# sourceMappingURL=tools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CACP,yJAAyJ,CAC1J;IACH,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,EAAE;SACL,WAAW,EAAE;SACb,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,aAAa,EAAE,CAAC;SACb,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gFAAgF,CAAC;CAC9F,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACzE,WAAW,EAAE,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;SACf,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,MAAM,EAAE,CAAC;SACN,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,wDAAwD,CAAC;IACrE,OAAO,EAAE,CAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,uBAAuB,CAAC;CACrC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC5D,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CAAC,wCAAwC,CAAC;IACrD,cAAc,EAAE,CAAC;SACd,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clearfetch",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"description": "
|
|
6
|
-
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Production-ready Model Context Protocol (MCP) server for ClearFetch (clearfetch.dev) - bypass Cloudflare, DataDome, and Turnstile to deliver clean Markdown & structured data to AI agents.",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"bin": {
|
|
9
|
+
"clearfetch": "bin/clearfetch-mcp.js",
|
|
10
|
+
"clearfetch-mcp": "bin/clearfetch-mcp.js"
|
|
11
|
+
},
|
|
12
|
+
"scripts": {
|
|
13
|
+
"build": "tsc",
|
|
14
|
+
"start": "node ./bin/clearfetch-mcp.js",
|
|
15
|
+
"dev": "tsc -w",
|
|
16
|
+
"prepublishOnly": "npm run build"
|
|
17
|
+
},
|
|
18
|
+
"keywords": [
|
|
19
|
+
"mcp",
|
|
20
|
+
"model-context-protocol",
|
|
21
|
+
"claude",
|
|
22
|
+
"cursor",
|
|
23
|
+
"windsurf",
|
|
24
|
+
"n8n",
|
|
25
|
+
"n8n-mcp",
|
|
26
|
+
"web-scraper",
|
|
27
|
+
"markdown",
|
|
28
|
+
"cloudflare-turnstile",
|
|
29
|
+
"datadome-bypass",
|
|
30
|
+
"ai-agent",
|
|
31
|
+
"rag"
|
|
32
|
+
],
|
|
33
|
+
"author": "ClearFetch Team <support@clearfetch.dev>",
|
|
34
|
+
"license": "MIT",
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@modelcontextprotocol/sdk": "^1.6.0",
|
|
37
|
+
"dotenv": "^16.4.7",
|
|
38
|
+
"zod": "^3.24.2"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"@types/node": "^22.13.9",
|
|
42
|
+
"typescript": "^5.8.2"
|
|
43
|
+
},
|
|
44
|
+
"files": [
|
|
45
|
+
"dist",
|
|
46
|
+
"bin",
|
|
47
|
+
"README.md"
|
|
48
|
+
]
|
|
49
|
+
}
|