clearfetch 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -2
- package/bin/unblocked-mcp.js +8 -0
- package/dist/client.d.ts +72 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +68 -0
- package/dist/client.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +184 -0
- package/dist/index.js.map +1 -0
- package/dist/tools.d.ts +50 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +61 -0
- package/dist/tools.js.map +1 -0
- package/package.json +46 -4
package/README.md
CHANGED
|
@@ -1,3 +1,125 @@
|
|
|
1
|
-
#
|
|
1
|
+
# clearfetch
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **The Model Context Protocol (MCP) Server for ClearFetch (`clearfetch.dev`)**
|
|
4
|
+
> Where Jina Reader and Firecrawl fail with `403 Forbidden` on Cloudflare Turnstile, DataDome, PerimeterX, or TLS/JA4 fingerprinting, ClearFetch guarantees clean, token-optimized Markdown and structured JSON extraction for AI agents.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## ⚡ Quick Start
|
|
9
|
+
|
|
10
|
+
Run directly via `npx` with zero installation:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
npx -y clearfetch
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
### Required Environment Variables
|
|
17
|
+
|
|
18
|
+
- `CLEARFETCH_API_KEY`: Your live API key from [ClearFetch Dashboard](https://clearfetch.dev/dashboard) (starts with `cf_live_...`).
|
|
19
|
+
- `CLEARFETCH_API_URL`: (Optional) Defaults to `https://api.clearfetch.dev`.
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## 🛠 Available Agent Tools
|
|
24
|
+
|
|
25
|
+
| Tool | Description | Key Parameters |
|
|
26
|
+
| :--- | :--- | :--- |
|
|
27
|
+
| `read_url` | Reads any URL (including Cloudflare/DataDome protected pages) as clean, token-optimized Markdown. | `url` (required), `stealth` ("auto" \| "force"), `max_age` (cache TTL in sec), `only_main_content` (bool) |
|
|
28
|
+
| `extract_structured_data` | Extracts typed JSON matching a strict schema from any webpage. | `url` (required), `json_schema` (required object), `prompt` (optional instruction) |
|
|
29
|
+
| `search_and_read` | Searches the web and returns full clean Markdown content of top results in one step. | `query` (required), `limit` (1-10, default 5), `scrape_results` (bool) |
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 💻 Client Configuration Examples
|
|
34
|
+
|
|
35
|
+
### 1. n8n (AI Agent Workflows)
|
|
36
|
+
|
|
37
|
+
In your n8n workflow with the **AI Agent** node:
|
|
38
|
+
1. Add the **Model Context Protocol (MCP) Tool** node connected to your AI Agent.
|
|
39
|
+
2. Configure Execution:
|
|
40
|
+
- **Command:** `npx`
|
|
41
|
+
- **Arguments:** `-y, clearfetch`
|
|
42
|
+
- **Environment Variables:**
|
|
43
|
+
- Key: `CLEARFETCH_API_KEY`, Value: `cf_live_your_api_key_here`
|
|
44
|
+
3. The AI Agent in n8n can now autonomously browse, unblock Turnstile-protected pages, and extract structured data!
|
|
45
|
+
|
|
46
|
+
*(Alternatively in n8n, you can use the standard **HTTP Request node** directly against `https://api.clearfetch.dev/v1/read`).*
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
### 2. Claude Desktop (`claude_desktop_config.json`)
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{
|
|
54
|
+
"mcpServers": {
|
|
55
|
+
"clearfetch": {
|
|
56
|
+
"command": "npx",
|
|
57
|
+
"args": ["-y", "clearfetch"],
|
|
58
|
+
"env": {
|
|
59
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
### 3. Cursor IDE (`.cursor/mcp.json`)
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{
|
|
72
|
+
"mcpServers": {
|
|
73
|
+
"clearfetch": {
|
|
74
|
+
"command": "npx",
|
|
75
|
+
"args": ["-y", "clearfetch"],
|
|
76
|
+
"env": {
|
|
77
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
### 4. Windsurf / Cline / Roo-Code
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{
|
|
90
|
+
"clearfetch": {
|
|
91
|
+
"command": "npx",
|
|
92
|
+
"args": ["-y", "clearfetch"],
|
|
93
|
+
"env": {
|
|
94
|
+
"CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
|
|
95
|
+
},
|
|
96
|
+
"disabled": false,
|
|
97
|
+
"autoApprove": [
|
|
98
|
+
"read_url",
|
|
99
|
+
"extract_structured_data",
|
|
100
|
+
"search_and_read"
|
|
101
|
+
]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 📦 Building from Source
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
cd packages/mcp-server
|
|
112
|
+
npm install
|
|
113
|
+
npm run build
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Verify stdio connectivity:
|
|
117
|
+
```bash
|
|
118
|
+
node ./bin/unblocked-mcp.js
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## 🛡 License
|
|
124
|
+
|
|
125
|
+
MIT License. Maintained by the ClearFetch Infrastructure Team (`clearfetch.dev`).
|
package/dist/client.d.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export interface ReadUrlParams {
|
|
2
|
+
url: string;
|
|
3
|
+
stealth?: "auto" | "force";
|
|
4
|
+
max_age?: number;
|
|
5
|
+
include_links?: boolean;
|
|
6
|
+
only_main_content?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface ReadUrlResponse {
|
|
9
|
+
success: boolean;
|
|
10
|
+
url: string;
|
|
11
|
+
title: string;
|
|
12
|
+
description: string;
|
|
13
|
+
markdown: string;
|
|
14
|
+
links: string[];
|
|
15
|
+
metadata: {
|
|
16
|
+
status_code: number;
|
|
17
|
+
cached: boolean;
|
|
18
|
+
stealth_used: boolean;
|
|
19
|
+
estimated_tokens: number;
|
|
20
|
+
latency_ms: number;
|
|
21
|
+
credits_used: number;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export interface ExtractDataParams {
|
|
25
|
+
url: string;
|
|
26
|
+
json_schema: Record<string, unknown>;
|
|
27
|
+
prompt?: string;
|
|
28
|
+
stealth?: "auto" | "force";
|
|
29
|
+
}
|
|
30
|
+
export interface ExtractDataResponse {
|
|
31
|
+
success: boolean;
|
|
32
|
+
url: string;
|
|
33
|
+
data: Record<string, unknown>;
|
|
34
|
+
metadata: {
|
|
35
|
+
status_code: number;
|
|
36
|
+
cached: boolean;
|
|
37
|
+
stealth_used: boolean;
|
|
38
|
+
estimated_tokens: number;
|
|
39
|
+
latency_ms: number;
|
|
40
|
+
credits_used: number;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export interface SearchAndReadParams {
|
|
44
|
+
query: string;
|
|
45
|
+
limit?: number;
|
|
46
|
+
scrape_results?: boolean;
|
|
47
|
+
}
|
|
48
|
+
export interface SearchResultItem {
|
|
49
|
+
title: string;
|
|
50
|
+
url: string;
|
|
51
|
+
snippet: string;
|
|
52
|
+
markdown?: string;
|
|
53
|
+
estimated_tokens?: number;
|
|
54
|
+
}
|
|
55
|
+
export interface SearchAndReadResponse {
|
|
56
|
+
success: boolean;
|
|
57
|
+
query: string;
|
|
58
|
+
count: number;
|
|
59
|
+
results: SearchResultItem[];
|
|
60
|
+
total_estimated_tokens: number;
|
|
61
|
+
credits_used: number;
|
|
62
|
+
}
|
|
63
|
+
export declare class UnblockedApiClient {
|
|
64
|
+
private baseUrl;
|
|
65
|
+
private apiKey;
|
|
66
|
+
constructor(apiKey?: string, baseUrl?: string);
|
|
67
|
+
private request;
|
|
68
|
+
readUrl(params: ReadUrlParams): Promise<ReadUrlResponse>;
|
|
69
|
+
extractData(params: ExtractDataParams): Promise<ExtractDataResponse>;
|
|
70
|
+
searchAndRead(params: SearchAndReadParams): Promise<SearchAndReadResponse>;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;YAW/B,OAAO;IAgCf,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAUxD,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IASpE,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAOjF"}
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UnblockedApiClient = void 0;
|
|
4
|
+
class UnblockedApiClient {
|
|
5
|
+
baseUrl;
|
|
6
|
+
apiKey;
|
|
7
|
+
constructor(apiKey, baseUrl) {
|
|
8
|
+
this.apiKey = apiKey || process.env.CLEARFETCH_API_KEY || process.env.UNBLOCKED_API_KEY || "";
|
|
9
|
+
this.baseUrl = (baseUrl || process.env.CLEARFETCH_API_URL || process.env.UNBLOCKED_API_URL || "https://api.clearfetch.dev").replace(/\/$/, "");
|
|
10
|
+
if (!this.apiKey) {
|
|
11
|
+
console.error("[WARN] CLEARFETCH_API_KEY environment variable is not set. MCP tools will fail authorization if not provided.");
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
async request(endpoint, body) {
|
|
15
|
+
const url = `${this.baseUrl}${endpoint}`;
|
|
16
|
+
const headers = {
|
|
17
|
+
"Content-Type": "application/json",
|
|
18
|
+
"User-Agent": "ClearFetch-MCP-Server/1.0.0",
|
|
19
|
+
};
|
|
20
|
+
if (this.apiKey) {
|
|
21
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
22
|
+
}
|
|
23
|
+
const response = await fetch(url, {
|
|
24
|
+
method: "POST",
|
|
25
|
+
headers,
|
|
26
|
+
body: JSON.stringify(body),
|
|
27
|
+
});
|
|
28
|
+
if (!response.ok) {
|
|
29
|
+
const errorText = await response.text();
|
|
30
|
+
let errorDetail = errorText;
|
|
31
|
+
try {
|
|
32
|
+
const errorJson = JSON.parse(errorText);
|
|
33
|
+
errorDetail = errorJson.detail || errorText;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// ignore json parse error
|
|
37
|
+
}
|
|
38
|
+
throw new Error(`Unblocked API Error (${response.status}): ${errorDetail}`);
|
|
39
|
+
}
|
|
40
|
+
return (await response.json());
|
|
41
|
+
}
|
|
42
|
+
async readUrl(params) {
|
|
43
|
+
return this.request("/v1/read", {
|
|
44
|
+
url: params.url,
|
|
45
|
+
stealth: params.stealth || "auto",
|
|
46
|
+
max_age: params.max_age ?? 3600,
|
|
47
|
+
include_links: params.include_links ?? true,
|
|
48
|
+
only_main_content: params.only_main_content ?? true,
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
async extractData(params) {
|
|
52
|
+
return this.request("/v1/extract", {
|
|
53
|
+
url: params.url,
|
|
54
|
+
json_schema: params.json_schema,
|
|
55
|
+
prompt: params.prompt,
|
|
56
|
+
stealth: params.stealth || "auto",
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
async searchAndRead(params) {
|
|
60
|
+
return this.request("/v1/search", {
|
|
61
|
+
query: params.query,
|
|
62
|
+
limit: params.limit ?? 5,
|
|
63
|
+
scrape_results: params.scrape_results ?? true,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.UnblockedApiClient = UnblockedApiClient;
|
|
68
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;AAqEA,MAAa,kBAAkB;IACrB,OAAO,CAAS;IAChB,MAAM,CAAS;IAEvB,YAAY,MAAe,EAAE,OAAgB;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;QAC9F,IAAI,CAAC,OAAO,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,4BAA4B,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAE/I,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,+GAA+G,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAI,QAAgB,EAAE,IAAa;QACtD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;QACzC,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;YAClC,YAAY,EAAE,6BAA6B;SAC5C,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,WAAW,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxC,WAAW,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,MAAM,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAqB;QACjC,OAAO,IAAI,CAAC,OAAO,CAAkB,UAAU,EAAE;YAC/C,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,IAAI;YAC3C,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;SACpD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAyB;QACzC,OAAO,IAAI,CAAC,OAAO,CAAsB,aAAa,EAAE;YACtD,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAA2B;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAwB,YAAY,EAAE;YACvD,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;SAC9C,CAAC,CAAC;IACL,CAAC;CACF;AAzED,gDAyEC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AA8GnE,wBAAgB,YAAY,IAAI,MAAM,CAoFrC;AAED,wBAAsB,cAAc,kBAKnC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.createServer = createServer;
|
|
7
|
+
exports.runStdioServer = runStdioServer;
|
|
8
|
+
const index_js_1 = require("@modelcontextprotocol/sdk/server/index.js");
|
|
9
|
+
const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
10
|
+
const types_js_1 = require("@modelcontextprotocol/sdk/types.js");
|
|
11
|
+
const client_js_1 = require("./client.js");
|
|
12
|
+
const tools_js_1 = require("./tools.js");
|
|
13
|
+
const dotenv_1 = __importDefault(require("dotenv"));
|
|
14
|
+
dotenv_1.default.config();
|
|
15
|
+
const client = new client_js_1.UnblockedApiClient();
|
|
16
|
+
const TOOLS = [
|
|
17
|
+
{
|
|
18
|
+
name: "read_url",
|
|
19
|
+
description: "Read any URL as clean, token-optimized Markdown. Reliably bypasses Cloudflare Turnstile, DataDome, PerimeterX, and anti-bot systems where other scrapers fail with 403 Forbidden.",
|
|
20
|
+
inputSchema: {
|
|
21
|
+
type: "object",
|
|
22
|
+
properties: {
|
|
23
|
+
url: {
|
|
24
|
+
type: "string",
|
|
25
|
+
description: "The full target URL to read (e.g. https://www.bloomberg.com/news/...)",
|
|
26
|
+
},
|
|
27
|
+
stealth: {
|
|
28
|
+
type: "string",
|
|
29
|
+
enum: ["auto", "force"],
|
|
30
|
+
default: "auto",
|
|
31
|
+
description: "'auto' attempts fast TLS spoofing first and auto-escalates on challenge detection; 'force' routes through full residential browser solver immediately.",
|
|
32
|
+
},
|
|
33
|
+
max_age: {
|
|
34
|
+
type: "integer",
|
|
35
|
+
default: 3600,
|
|
36
|
+
description: "Cache TTL in seconds. 0 forces live fresh fetch.",
|
|
37
|
+
},
|
|
38
|
+
include_links: {
|
|
39
|
+
type: "boolean",
|
|
40
|
+
default: true,
|
|
41
|
+
description: "Whether to return extracted outgoing links.",
|
|
42
|
+
},
|
|
43
|
+
only_main_content: {
|
|
44
|
+
type: "boolean",
|
|
45
|
+
default: true,
|
|
46
|
+
description: "Whether to strip navigation, ads, and footers.",
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
required: ["url"],
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
name: "extract_structured_data",
|
|
54
|
+
description: "Extract strongly-typed JSON matching a strict schema from any webpage, even behind heavy anti-bot protections.",
|
|
55
|
+
inputSchema: {
|
|
56
|
+
type: "object",
|
|
57
|
+
properties: {
|
|
58
|
+
url: {
|
|
59
|
+
type: "string",
|
|
60
|
+
description: "Target webpage URL to extract data from",
|
|
61
|
+
},
|
|
62
|
+
json_schema: {
|
|
63
|
+
type: "object",
|
|
64
|
+
description: "Strict JSON Schema describing the target data fields",
|
|
65
|
+
},
|
|
66
|
+
prompt: {
|
|
67
|
+
type: "string",
|
|
68
|
+
description: "Optional custom instruction or focus for extraction",
|
|
69
|
+
},
|
|
70
|
+
stealth: {
|
|
71
|
+
type: "string",
|
|
72
|
+
enum: ["auto", "force"],
|
|
73
|
+
default: "auto",
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
required: ["url", "json_schema"],
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
name: "search_and_read",
|
|
81
|
+
description: "Search the live web and return clean Markdown content bundles of top results in a single step for AI agent context / RAG.",
|
|
82
|
+
inputSchema: {
|
|
83
|
+
type: "object",
|
|
84
|
+
properties: {
|
|
85
|
+
query: {
|
|
86
|
+
type: "string",
|
|
87
|
+
description: "Search query string",
|
|
88
|
+
},
|
|
89
|
+
limit: {
|
|
90
|
+
type: "integer",
|
|
91
|
+
default: 5,
|
|
92
|
+
minimum: 1,
|
|
93
|
+
maximum: 10,
|
|
94
|
+
description: "Number of search results to return (1-10)",
|
|
95
|
+
},
|
|
96
|
+
scrape_results: {
|
|
97
|
+
type: "boolean",
|
|
98
|
+
default: true,
|
|
99
|
+
description: "Whether to fetch full clean Markdown for each result",
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
required: ["query"],
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
];
|
|
106
|
+
function createServer() {
|
|
107
|
+
const server = new index_js_1.Server({
|
|
108
|
+
name: "unblocked-web-mcp",
|
|
109
|
+
version: "1.0.0",
|
|
110
|
+
}, {
|
|
111
|
+
capabilities: {
|
|
112
|
+
tools: {},
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
|
|
116
|
+
return { tools: TOOLS };
|
|
117
|
+
});
|
|
118
|
+
server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
|
|
119
|
+
const { name, arguments: args } = request.params;
|
|
120
|
+
try {
|
|
121
|
+
if (name === "read_url") {
|
|
122
|
+
const validated = tools_js_1.ReadUrlSchema.parse(args);
|
|
123
|
+
const res = await client.readUrl(validated);
|
|
124
|
+
return {
|
|
125
|
+
content: [
|
|
126
|
+
{
|
|
127
|
+
type: "text",
|
|
128
|
+
text: `# ${res.title}\n\n${res.markdown}\n\n---\n*Metadata: status=${res.metadata.status_code}, cached=${res.metadata.cached}, stealth=${res.metadata.stealth_used}, tokens=${res.metadata.estimated_tokens}, latency=${res.metadata.latency_ms}ms*`,
|
|
129
|
+
},
|
|
130
|
+
],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
if (name === "extract_structured_data") {
|
|
134
|
+
const validated = tools_js_1.ExtractDataSchema.parse(args);
|
|
135
|
+
const res = await client.extractData(validated);
|
|
136
|
+
return {
|
|
137
|
+
content: [
|
|
138
|
+
{
|
|
139
|
+
type: "text",
|
|
140
|
+
text: JSON.stringify(res.data, null, 2),
|
|
141
|
+
},
|
|
142
|
+
],
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (name === "search_and_read") {
|
|
146
|
+
const validated = tools_js_1.SearchAndReadSchema.parse(args);
|
|
147
|
+
const res = await client.searchAndRead(validated);
|
|
148
|
+
const formattedResults = res.results
|
|
149
|
+
.map((item, index) => {
|
|
150
|
+
return `### [${index + 1}] ${item.title}\nURL: ${item.url}\n${item.markdown ? `\nContent:\n${item.markdown}\n` : `\nSnippet: ${item.snippet}\n`}`;
|
|
151
|
+
})
|
|
152
|
+
.join("\n---\n\n");
|
|
153
|
+
return {
|
|
154
|
+
content: [
|
|
155
|
+
{
|
|
156
|
+
type: "text",
|
|
157
|
+
text: `# Search Results for: "${res.query}"\n\n${formattedResults}\n\n---\n*Total tokens: ${res.total_estimated_tokens}, Results count: ${res.count}*`,
|
|
158
|
+
},
|
|
159
|
+
],
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
throw new Error(`Unknown tool requested: ${name}`);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
return {
|
|
166
|
+
isError: true,
|
|
167
|
+
content: [
|
|
168
|
+
{
|
|
169
|
+
type: "text",
|
|
170
|
+
text: `Error executing ${name}: ${error.message || String(error)}`,
|
|
171
|
+
},
|
|
172
|
+
],
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
return server;
|
|
177
|
+
}
|
|
178
|
+
async function runStdioServer() {
|
|
179
|
+
const server = createServer();
|
|
180
|
+
const transport = new stdio_js_1.StdioServerTransport();
|
|
181
|
+
await server.connect(transport);
|
|
182
|
+
console.error("Unblocked Web MCP Server running on stdio transport.");
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;AA8GA,oCAoFC;AAED,wCAKC;AAzMD,wEAAmE;AACnE,wEAAiF;AACjF,iEAI4C;AAC5C,2CAAiD;AACjD,yCAAmF;AACnF,oDAA4B;AAE5B,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,MAAM,GAAG,IAAI,8BAAkB,EAAE,CAAC;AAExC,MAAM,KAAK,GAAW;IACpB;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EACT,mLAAmL;QACrL,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uEAAuE;iBACrF;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;oBACf,WAAW,EACT,wJAAwJ;iBAC3J;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,kDAAkD;iBAChE;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,6CAA6C;iBAC3D;gBACD,iBAAiB,EAAE;oBACjB,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,gDAAgD;iBAC9D;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;IACD;QACE,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EACT,gHAAgH;QAClH,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,yCAAyC;iBACvD;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,sDAAsD;iBACpE;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qDAAqD;iBACnE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;iBAChB;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC;SACjC;KACF;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EACT,2HAA2H;QAC7H,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qBAAqB;iBACnC;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;oBACX,WAAW,EAAE,2CAA2C;iBACzD;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,sDAAsD;iBACpE;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;KACF;CACF,CAAC;AAEF,SAAgB,YAAY;IAC1B,MAAM,MAAM,GAAG,IAAI,iBAAM,CACvB;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE;SACV;KACF,CACF,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,iCAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,gCAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAEjD,IAAI,CAAC;YACH,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;gBACxB,MAAM,SAAS,GAAG,wBAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC5C,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,KAAK,GAAG,CAAC,KAAK,OAAO,GAAG,CAAC,QAAQ,8BAA8B,GAAG,CAAC,QAAQ,CAAC,WAAW,YAAY,GAAG,CAAC,QAAQ,CAAC,MAAM,aAAa,GAAG,CAAC,QAAQ,CAAC,YAAY,YAAY,GAAG,CAAC,QAAQ,CAAC,gBAAgB,aAAa,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK;yBACrP;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,yBAAyB,EAAE,CAAC;gBACvC,MAAM,SAAS,GAAG,4BAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAChD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;yBACxC;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBAC/B,MAAM,SAAS,GAAG,8BAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBAElD,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO;qBACjC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBACnB,OAAO,QAAQ,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,KACvD,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,OAAO,IAC/E,EAAE,CAAC;gBACL,CAAC,CAAC;qBACD,IAAI,CAAC,WAAW,CAAC,CAAC;gBAErB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,0BAA0B,GAAG,CAAC,KAAK,QAAQ,gBAAgB,2BAA2B,GAAG,CAAC,sBAAsB,oBAAoB,GAAG,CAAC,KAAK,GAAG;yBACvJ;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,mBAAmB,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE;qBACnE;iBACF;aACF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,cAAc;IAClC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,+BAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;AACxE,CAAC"}
|
package/dist/tools.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const ReadUrlSchema: z.ZodObject<{
|
|
3
|
+
url: z.ZodString;
|
|
4
|
+
stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
|
|
5
|
+
max_age: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
6
|
+
include_links: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
7
|
+
only_main_content: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
8
|
+
}, "strip", z.ZodTypeAny, {
|
|
9
|
+
url: string;
|
|
10
|
+
stealth: "auto" | "force";
|
|
11
|
+
max_age: number;
|
|
12
|
+
include_links: boolean;
|
|
13
|
+
only_main_content: boolean;
|
|
14
|
+
}, {
|
|
15
|
+
url: string;
|
|
16
|
+
stealth?: "auto" | "force" | undefined;
|
|
17
|
+
max_age?: number | undefined;
|
|
18
|
+
include_links?: boolean | undefined;
|
|
19
|
+
only_main_content?: boolean | undefined;
|
|
20
|
+
}>;
|
|
21
|
+
export declare const ExtractDataSchema: z.ZodObject<{
|
|
22
|
+
url: z.ZodString;
|
|
23
|
+
json_schema: z.ZodRecord<z.ZodString, z.ZodAny>;
|
|
24
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
25
|
+
stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
|
|
26
|
+
}, "strip", z.ZodTypeAny, {
|
|
27
|
+
url: string;
|
|
28
|
+
stealth: "auto" | "force";
|
|
29
|
+
json_schema: Record<string, any>;
|
|
30
|
+
prompt?: string | undefined;
|
|
31
|
+
}, {
|
|
32
|
+
url: string;
|
|
33
|
+
json_schema: Record<string, any>;
|
|
34
|
+
stealth?: "auto" | "force" | undefined;
|
|
35
|
+
prompt?: string | undefined;
|
|
36
|
+
}>;
|
|
37
|
+
export declare const SearchAndReadSchema: z.ZodObject<{
|
|
38
|
+
query: z.ZodString;
|
|
39
|
+
limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
40
|
+
scrape_results: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
41
|
+
}, "strip", z.ZodTypeAny, {
|
|
42
|
+
query: string;
|
|
43
|
+
limit: number;
|
|
44
|
+
scrape_results: boolean;
|
|
45
|
+
}, {
|
|
46
|
+
query: string;
|
|
47
|
+
limit?: number | undefined;
|
|
48
|
+
scrape_results?: boolean | undefined;
|
|
49
|
+
}>;
|
|
50
|
+
//# sourceMappingURL=tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;EA0BxB,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;EAc5B,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;EAe9B,CAAC"}
|
package/dist/tools.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SearchAndReadSchema = exports.ExtractDataSchema = exports.ReadUrlSchema = void 0;
|
|
4
|
+
const zod_1 = require("zod");
|
|
5
|
+
exports.ReadUrlSchema = zod_1.z.object({
|
|
6
|
+
url: zod_1.z.string().url().describe("The full web URL to scrape and read into clean Markdown"),
|
|
7
|
+
stealth: zod_1.z
|
|
8
|
+
.enum(["auto", "force"])
|
|
9
|
+
.optional()
|
|
10
|
+
.default("auto")
|
|
11
|
+
.describe("'auto' tries fast TLS spoofing first and auto-escalates on 403/Turnstile; 'force' routes through full residential proxy + Turnstile solver immediately."),
|
|
12
|
+
max_age: zod_1.z
|
|
13
|
+
.number()
|
|
14
|
+
.int()
|
|
15
|
+
.nonnegative()
|
|
16
|
+
.optional()
|
|
17
|
+
.default(3600)
|
|
18
|
+
.describe("Redis cache TTL in seconds. Use 0 to force a real-time fresh bypass fetch."),
|
|
19
|
+
include_links: zod_1.z
|
|
20
|
+
.boolean()
|
|
21
|
+
.optional()
|
|
22
|
+
.default(true)
|
|
23
|
+
.describe("Whether to include extracted outgoing links in the response"),
|
|
24
|
+
only_main_content: zod_1.z
|
|
25
|
+
.boolean()
|
|
26
|
+
.optional()
|
|
27
|
+
.default(true)
|
|
28
|
+
.describe("Whether to strip navigational headers, footers, and ad noise using Trafilatura"),
|
|
29
|
+
});
|
|
30
|
+
exports.ExtractDataSchema = zod_1.z.object({
|
|
31
|
+
url: zod_1.z.string().url().describe("Target webpage URL to extract data from"),
|
|
32
|
+
json_schema: zod_1.z
|
|
33
|
+
.record(zod_1.z.any())
|
|
34
|
+
.describe("JSON Schema definition defining the desired typed structure"),
|
|
35
|
+
prompt: zod_1.z
|
|
36
|
+
.string()
|
|
37
|
+
.optional()
|
|
38
|
+
.describe("Optional extraction prompt or instructions for the LLM"),
|
|
39
|
+
stealth: zod_1.z
|
|
40
|
+
.enum(["auto", "force"])
|
|
41
|
+
.optional()
|
|
42
|
+
.default("auto")
|
|
43
|
+
.describe("Anti-bot bypass level"),
|
|
44
|
+
});
|
|
45
|
+
exports.SearchAndReadSchema = zod_1.z.object({
|
|
46
|
+
query: zod_1.z.string().min(1).describe("Search query to execute"),
|
|
47
|
+
limit: zod_1.z
|
|
48
|
+
.number()
|
|
49
|
+
.int()
|
|
50
|
+
.min(1)
|
|
51
|
+
.max(10)
|
|
52
|
+
.optional()
|
|
53
|
+
.default(5)
|
|
54
|
+
.describe("Number of top results to return (1-10)"),
|
|
55
|
+
scrape_results: zod_1.z
|
|
56
|
+
.boolean()
|
|
57
|
+
.optional()
|
|
58
|
+
.default(true)
|
|
59
|
+
.describe("Whether to concurrently fetch and convert each search result into clean Markdown"),
|
|
60
|
+
});
|
|
61
|
+
//# sourceMappingURL=tools.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAEX,QAAA,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IACzF,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CACP,yJAAyJ,CAC1J;IACH,OAAO,EAAE,OAAC;SACP,MAAM,EAAE;SACR,GAAG,EAAE;SACL,WAAW,EAAE;SACb,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,aAAa,EAAE,OAAC;SACb,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,iBAAiB,EAAE,OAAC;SACjB,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gFAAgF,CAAC;CAC9F,CAAC,CAAC;AAEU,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACzE,WAAW,EAAE,OAAC;SACX,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC;SACf,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,wDAAwD,CAAC;IACrE,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,uBAAuB,CAAC;CACrC,CAAC,CAAC;AAEU,QAAA,mBAAmB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC1C,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC5D,KAAK,EAAE,OAAC;SACL,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CAAC,wCAAwC,CAAC;IACrD,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,48 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clearfetch",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Production-ready Model Context Protocol (MCP) server for ClearFetch (clearfetch.dev) - bypass Cloudflare, DataDome, and Turnstile to deliver clean Markdown & structured data to AI agents.",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"types": "dist/index.d.ts",
|
|
7
|
+
"bin": {
|
|
8
|
+
"clearfetch": "./bin/unblocked-mcp.js",
|
|
9
|
+
"clearfetch-mcp": "./bin/unblocked-mcp.js"
|
|
10
|
+
},
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc",
|
|
13
|
+
"start": "node ./bin/unblocked-mcp.js",
|
|
14
|
+
"dev": "tsc -w",
|
|
15
|
+
"prepublishOnly": "npm run build"
|
|
16
|
+
},
|
|
17
|
+
"keywords": [
|
|
18
|
+
"mcp",
|
|
19
|
+
"model-context-protocol",
|
|
20
|
+
"claude",
|
|
21
|
+
"cursor",
|
|
22
|
+
"windsurf",
|
|
23
|
+
"n8n",
|
|
24
|
+
"n8n-mcp",
|
|
25
|
+
"web-scraper",
|
|
26
|
+
"markdown",
|
|
27
|
+
"cloudflare-turnstile",
|
|
28
|
+
"datadome-bypass",
|
|
29
|
+
"ai-agent",
|
|
30
|
+
"rag"
|
|
31
|
+
],
|
|
32
|
+
"author": "ClearFetch Team <support@clearfetch.dev>",
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@modelcontextprotocol/sdk": "^1.6.0",
|
|
36
|
+
"dotenv": "^16.4.7",
|
|
37
|
+
"zod": "^3.24.2"
|
|
38
|
+
},
|
|
39
|
+
"devDependencies": {
|
|
40
|
+
"@types/node": "^22.13.9",
|
|
41
|
+
"typescript": "^5.8.2"
|
|
42
|
+
},
|
|
43
|
+
"files": [
|
|
44
|
+
"dist",
|
|
45
|
+
"bin",
|
|
46
|
+
"README.md"
|
|
47
|
+
]
|
|
48
|
+
}
|