clearfetch 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,3 +1,125 @@
1
- # Temporary Holding Version
1
+ # clearfetch
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ > **The Model Context Protocol (MCP) Server for ClearFetch (`clearfetch.dev`)**
4
+ > Where Jina Reader and Firecrawl fail with `403 Forbidden` on Cloudflare Turnstile, DataDome, PerimeterX, or TLS/JA4 fingerprinting, ClearFetch guarantees clean, token-optimized Markdown and structured JSON extraction for AI agents.
5
+
6
+ ---
7
+
8
+ ## ⚡ Quick Start
9
+
10
+ Run directly via `npx` with zero installation:
11
+
12
+ ```bash
13
+ npx -y clearfetch
14
+ ```
15
+
16
+ ### Required Environment Variables
17
+
18
+ - `CLEARFETCH_API_KEY`: Your live API key from [ClearFetch Dashboard](https://clearfetch.dev/dashboard) (starts with `cf_live_...`).
19
+ - `CLEARFETCH_API_URL`: (Optional) Defaults to `https://api.clearfetch.dev`.
20
+
21
+ ---
22
+
23
+ ## 🛠 Available Agent Tools
24
+
25
+ | Tool | Description | Key Parameters |
26
+ | :--- | :--- | :--- |
27
+ | `read_url` | Reads any URL (including Cloudflare/DataDome protected pages) as clean, token-optimized Markdown. | `url` (required), `stealth` ("auto" \| "force"), `max_age` (cache TTL in sec), `only_main_content` (bool) |
28
+ | `extract_structured_data` | Extracts typed JSON matching a strict schema from any webpage. | `url` (required), `json_schema` (required object), `prompt` (optional instruction) |
29
+ | `search_and_read` | Searches the web and returns full clean Markdown content of top results in one step. | `query` (required), `limit` (1-10, default 5), `scrape_results` (bool) |
30
+
31
+ ---
32
+
33
+ ## 💻 Client Configuration Examples
34
+
35
+ ### 1. n8n (AI Agent Workflows)
36
+
37
+ In your n8n workflow with the **AI Agent** node:
38
+ 1. Add the **Model Context Protocol (MCP) Tool** node connected to your AI Agent.
39
+ 2. Configure Execution:
40
+ - **Command:** `npx`
41
+ - **Arguments:** `-y, clearfetch`
42
+ - **Environment Variables:**
43
+ - Key: `CLEARFETCH_API_KEY`, Value: `cf_live_your_api_key_here`
44
+ 3. The AI Agent in n8n can now autonomously browse, unblock Turnstile-protected pages, and extract structured data!
45
+
46
+ *(Alternatively in n8n, you can use the standard **HTTP Request node** directly against `https://api.clearfetch.dev/v1/read`).*
47
+
48
+ ---
49
+
50
+ ### 2. Claude Desktop (`claude_desktop_config.json`)
51
+
52
+ ```json
53
+ {
54
+ "mcpServers": {
55
+ "clearfetch": {
56
+ "command": "npx",
57
+ "args": ["-y", "clearfetch"],
58
+ "env": {
59
+ "CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
60
+ }
61
+ }
62
+ }
63
+ }
64
+ ```
65
+
66
+ ---
67
+
68
+ ### 3. Cursor IDE (`.cursor/mcp.json`)
69
+
70
+ ```json
71
+ {
72
+ "mcpServers": {
73
+ "clearfetch": {
74
+ "command": "npx",
75
+ "args": ["-y", "clearfetch"],
76
+ "env": {
77
+ "CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
78
+ }
79
+ }
80
+ }
81
+ }
82
+ ```
83
+
84
+ ---
85
+
86
+ ### 4. Windsurf / Cline / Roo-Code
87
+
88
+ ```json
89
+ {
90
+ "clearfetch": {
91
+ "command": "npx",
92
+ "args": ["-y", "clearfetch"],
93
+ "env": {
94
+ "CLEARFETCH_API_KEY": "cf_live_your_api_key_here"
95
+ },
96
+ "disabled": false,
97
+ "autoApprove": [
98
+ "read_url",
99
+ "extract_structured_data",
100
+ "search_and_read"
101
+ ]
102
+ }
103
+ }
104
+ ```
105
+
106
+ ---
107
+
108
+ ## 📦 Building from Source
109
+
110
+ ```bash
111
+ cd packages/mcp-server
112
+ npm install
113
+ npm run build
114
+ ```
115
+
116
+ Verify stdio connectivity:
117
+ ```bash
118
+ node ./bin/unblocked-mcp.js
119
+ ```
120
+
121
+ ---
122
+
123
+ ## 🛡 License
124
+
125
+ MIT License. Maintained by the ClearFetch Infrastructure Team (`clearfetch.dev`).
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env node
2
+
3
+ import { runStdioServer } from "../dist/index.js";
4
+
5
+ runStdioServer().catch((error) => {
6
+ console.error("Fatal error starting Unblocked Web MCP Server:", error);
7
+ process.exit(1);
8
+ });
@@ -0,0 +1,72 @@
1
+ export interface ReadUrlParams {
2
+ url: string;
3
+ stealth?: "auto" | "force";
4
+ max_age?: number;
5
+ include_links?: boolean;
6
+ only_main_content?: boolean;
7
+ }
8
+ export interface ReadUrlResponse {
9
+ success: boolean;
10
+ url: string;
11
+ title: string;
12
+ description: string;
13
+ markdown: string;
14
+ links: string[];
15
+ metadata: {
16
+ status_code: number;
17
+ cached: boolean;
18
+ stealth_used: boolean;
19
+ estimated_tokens: number;
20
+ latency_ms: number;
21
+ credits_used: number;
22
+ };
23
+ }
24
+ export interface ExtractDataParams {
25
+ url: string;
26
+ json_schema: Record<string, unknown>;
27
+ prompt?: string;
28
+ stealth?: "auto" | "force";
29
+ }
30
+ export interface ExtractDataResponse {
31
+ success: boolean;
32
+ url: string;
33
+ data: Record<string, unknown>;
34
+ metadata: {
35
+ status_code: number;
36
+ cached: boolean;
37
+ stealth_used: boolean;
38
+ estimated_tokens: number;
39
+ latency_ms: number;
40
+ credits_used: number;
41
+ };
42
+ }
43
+ export interface SearchAndReadParams {
44
+ query: string;
45
+ limit?: number;
46
+ scrape_results?: boolean;
47
+ }
48
+ export interface SearchResultItem {
49
+ title: string;
50
+ url: string;
51
+ snippet: string;
52
+ markdown?: string;
53
+ estimated_tokens?: number;
54
+ }
55
+ export interface SearchAndReadResponse {
56
+ success: boolean;
57
+ query: string;
58
+ count: number;
59
+ results: SearchResultItem[];
60
+ total_estimated_tokens: number;
61
+ credits_used: number;
62
+ }
63
+ export declare class UnblockedApiClient {
64
+ private baseUrl;
65
+ private apiKey;
66
+ constructor(apiKey?: string, baseUrl?: string);
67
+ private request;
68
+ readUrl(params: ReadUrlParams): Promise<ReadUrlResponse>;
69
+ extractData(params: ExtractDataParams): Promise<ExtractDataResponse>;
70
+ searchAndRead(params: SearchAndReadParams): Promise<SearchAndReadResponse>;
71
+ }
72
+ //# sourceMappingURL=client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,OAAO,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;YAW/B,OAAO;IAgCf,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAUxD,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IASpE,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAOjF"}
package/dist/client.js ADDED
@@ -0,0 +1,68 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.UnblockedApiClient = void 0;
4
+ class UnblockedApiClient {
5
+ baseUrl;
6
+ apiKey;
7
+ constructor(apiKey, baseUrl) {
8
+ this.apiKey = apiKey || process.env.CLEARFETCH_API_KEY || process.env.UNBLOCKED_API_KEY || "";
9
+ this.baseUrl = (baseUrl || process.env.CLEARFETCH_API_URL || process.env.UNBLOCKED_API_URL || "https://api.clearfetch.dev").replace(/\/$/, "");
10
+ if (!this.apiKey) {
11
+ console.error("[WARN] CLEARFETCH_API_KEY environment variable is not set. MCP tools will fail authorization if not provided.");
12
+ }
13
+ }
14
+ async request(endpoint, body) {
15
+ const url = `${this.baseUrl}${endpoint}`;
16
+ const headers = {
17
+ "Content-Type": "application/json",
18
+ "User-Agent": "ClearFetch-MCP-Server/1.0.0",
19
+ };
20
+ if (this.apiKey) {
21
+ headers["Authorization"] = `Bearer ${this.apiKey}`;
22
+ }
23
+ const response = await fetch(url, {
24
+ method: "POST",
25
+ headers,
26
+ body: JSON.stringify(body),
27
+ });
28
+ if (!response.ok) {
29
+ const errorText = await response.text();
30
+ let errorDetail = errorText;
31
+ try {
32
+ const errorJson = JSON.parse(errorText);
33
+ errorDetail = errorJson.detail || errorText;
34
+ }
35
+ catch {
36
+ // ignore json parse error
37
+ }
38
+ throw new Error(`Unblocked API Error (${response.status}): ${errorDetail}`);
39
+ }
40
+ return (await response.json());
41
+ }
42
+ async readUrl(params) {
43
+ return this.request("/v1/read", {
44
+ url: params.url,
45
+ stealth: params.stealth || "auto",
46
+ max_age: params.max_age ?? 3600,
47
+ include_links: params.include_links ?? true,
48
+ only_main_content: params.only_main_content ?? true,
49
+ });
50
+ }
51
+ async extractData(params) {
52
+ return this.request("/v1/extract", {
53
+ url: params.url,
54
+ json_schema: params.json_schema,
55
+ prompt: params.prompt,
56
+ stealth: params.stealth || "auto",
57
+ });
58
+ }
59
+ async searchAndRead(params) {
60
+ return this.request("/v1/search", {
61
+ query: params.query,
62
+ limit: params.limit ?? 5,
63
+ scrape_results: params.scrape_results ?? true,
64
+ });
65
+ }
66
+ }
67
+ exports.UnblockedApiClient = UnblockedApiClient;
68
+ //# sourceMappingURL=client.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;AAqEA,MAAa,kBAAkB;IACrB,OAAO,CAAS;IAChB,MAAM,CAAS;IAEvB,YAAY,MAAe,EAAE,OAAgB;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;QAC9F,IAAI,CAAC,OAAO,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,4BAA4B,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAE/I,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,+GAA+G,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAI,QAAgB,EAAE,IAAa;QACtD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;QACzC,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;YAClC,YAAY,EAAE,6BAA6B;SAC5C,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,IAAI,WAAW,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACxC,WAAW,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,MAAM,WAAW,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAqB;QACjC,OAAO,IAAI,CAAC,OAAO,CAAkB,UAAU,EAAE;YAC/C,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,IAAI;YAC3C,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;SACpD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAyB;QACzC,OAAO,IAAI,CAAC,OAAO,CAAsB,aAAa,EAAE;YACtD,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,MAAM;SAClC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAA2B;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAwB,YAAY,EAAE;YACvD,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;SAC9C,CAAC,CAAC;IACL,CAAC;CACF;AAzED,gDAyEC"}
@@ -0,0 +1,4 @@
1
+ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
2
+ export declare function createServer(): Server;
3
+ export declare function runStdioServer(): Promise<void>;
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AA8GnE,wBAAgB,YAAY,IAAI,MAAM,CAoFrC;AAED,wBAAsB,cAAc,kBAKnC"}
package/dist/index.js ADDED
@@ -0,0 +1,184 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.createServer = createServer;
7
+ exports.runStdioServer = runStdioServer;
8
+ const index_js_1 = require("@modelcontextprotocol/sdk/server/index.js");
9
+ const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
10
+ const types_js_1 = require("@modelcontextprotocol/sdk/types.js");
11
+ const client_js_1 = require("./client.js");
12
+ const tools_js_1 = require("./tools.js");
13
+ const dotenv_1 = __importDefault(require("dotenv"));
14
+ dotenv_1.default.config();
15
+ const client = new client_js_1.UnblockedApiClient();
16
+ const TOOLS = [
17
+ {
18
+ name: "read_url",
19
+ description: "Read any URL as clean, token-optimized Markdown. Reliably bypasses Cloudflare Turnstile, DataDome, PerimeterX, and anti-bot systems where other scrapers fail with 403 Forbidden.",
20
+ inputSchema: {
21
+ type: "object",
22
+ properties: {
23
+ url: {
24
+ type: "string",
25
+ description: "The full target URL to read (e.g. https://www.bloomberg.com/news/...)",
26
+ },
27
+ stealth: {
28
+ type: "string",
29
+ enum: ["auto", "force"],
30
+ default: "auto",
31
+ description: "'auto' attempts fast TLS spoofing first and auto-escalates on challenge detection; 'force' routes through full residential browser solver immediately.",
32
+ },
33
+ max_age: {
34
+ type: "integer",
35
+ default: 3600,
36
+ description: "Cache TTL in seconds. 0 forces live fresh fetch.",
37
+ },
38
+ include_links: {
39
+ type: "boolean",
40
+ default: true,
41
+ description: "Whether to return extracted outgoing links.",
42
+ },
43
+ only_main_content: {
44
+ type: "boolean",
45
+ default: true,
46
+ description: "Whether to strip navigation, ads, and footers.",
47
+ },
48
+ },
49
+ required: ["url"],
50
+ },
51
+ },
52
+ {
53
+ name: "extract_structured_data",
54
+ description: "Extract strongly-typed JSON matching a strict schema from any webpage, even behind heavy anti-bot protections.",
55
+ inputSchema: {
56
+ type: "object",
57
+ properties: {
58
+ url: {
59
+ type: "string",
60
+ description: "Target webpage URL to extract data from",
61
+ },
62
+ json_schema: {
63
+ type: "object",
64
+ description: "Strict JSON Schema describing the target data fields",
65
+ },
66
+ prompt: {
67
+ type: "string",
68
+ description: "Optional custom instruction or focus for extraction",
69
+ },
70
+ stealth: {
71
+ type: "string",
72
+ enum: ["auto", "force"],
73
+ default: "auto",
74
+ },
75
+ },
76
+ required: ["url", "json_schema"],
77
+ },
78
+ },
79
+ {
80
+ name: "search_and_read",
81
+ description: "Search the live web and return clean Markdown content bundles of top results in a single step for AI agent context / RAG.",
82
+ inputSchema: {
83
+ type: "object",
84
+ properties: {
85
+ query: {
86
+ type: "string",
87
+ description: "Search query string",
88
+ },
89
+ limit: {
90
+ type: "integer",
91
+ default: 5,
92
+ minimum: 1,
93
+ maximum: 10,
94
+ description: "Number of search results to return (1-10)",
95
+ },
96
+ scrape_results: {
97
+ type: "boolean",
98
+ default: true,
99
+ description: "Whether to fetch full clean Markdown for each result",
100
+ },
101
+ },
102
+ required: ["query"],
103
+ },
104
+ },
105
+ ];
106
+ function createServer() {
107
+ const server = new index_js_1.Server({
108
+ name: "unblocked-web-mcp",
109
+ version: "1.0.0",
110
+ }, {
111
+ capabilities: {
112
+ tools: {},
113
+ },
114
+ });
115
+ server.setRequestHandler(types_js_1.ListToolsRequestSchema, async () => {
116
+ return { tools: TOOLS };
117
+ });
118
+ server.setRequestHandler(types_js_1.CallToolRequestSchema, async (request) => {
119
+ const { name, arguments: args } = request.params;
120
+ try {
121
+ if (name === "read_url") {
122
+ const validated = tools_js_1.ReadUrlSchema.parse(args);
123
+ const res = await client.readUrl(validated);
124
+ return {
125
+ content: [
126
+ {
127
+ type: "text",
128
+ text: `# ${res.title}\n\n${res.markdown}\n\n---\n*Metadata: status=${res.metadata.status_code}, cached=${res.metadata.cached}, stealth=${res.metadata.stealth_used}, tokens=${res.metadata.estimated_tokens}, latency=${res.metadata.latency_ms}ms*`,
129
+ },
130
+ ],
131
+ };
132
+ }
133
+ if (name === "extract_structured_data") {
134
+ const validated = tools_js_1.ExtractDataSchema.parse(args);
135
+ const res = await client.extractData(validated);
136
+ return {
137
+ content: [
138
+ {
139
+ type: "text",
140
+ text: JSON.stringify(res.data, null, 2),
141
+ },
142
+ ],
143
+ };
144
+ }
145
+ if (name === "search_and_read") {
146
+ const validated = tools_js_1.SearchAndReadSchema.parse(args);
147
+ const res = await client.searchAndRead(validated);
148
+ const formattedResults = res.results
149
+ .map((item, index) => {
150
+ return `### [${index + 1}] ${item.title}\nURL: ${item.url}\n${item.markdown ? `\nContent:\n${item.markdown}\n` : `\nSnippet: ${item.snippet}\n`}`;
151
+ })
152
+ .join("\n---\n\n");
153
+ return {
154
+ content: [
155
+ {
156
+ type: "text",
157
+ text: `# Search Results for: "${res.query}"\n\n${formattedResults}\n\n---\n*Total tokens: ${res.total_estimated_tokens}, Results count: ${res.count}*`,
158
+ },
159
+ ],
160
+ };
161
+ }
162
+ throw new Error(`Unknown tool requested: ${name}`);
163
+ }
164
+ catch (error) {
165
+ return {
166
+ isError: true,
167
+ content: [
168
+ {
169
+ type: "text",
170
+ text: `Error executing ${name}: ${error.message || String(error)}`,
171
+ },
172
+ ],
173
+ };
174
+ }
175
+ });
176
+ return server;
177
+ }
178
+ async function runStdioServer() {
179
+ const server = createServer();
180
+ const transport = new stdio_js_1.StdioServerTransport();
181
+ await server.connect(transport);
182
+ console.error("Unblocked Web MCP Server running on stdio transport.");
183
+ }
184
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;AA8GA,oCAoFC;AAED,wCAKC;AAzMD,wEAAmE;AACnE,wEAAiF;AACjF,iEAI4C;AAC5C,2CAAiD;AACjD,yCAAmF;AACnF,oDAA4B;AAE5B,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,MAAM,GAAG,IAAI,8BAAkB,EAAE,CAAC;AAExC,MAAM,KAAK,GAAW;IACpB;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EACT,mLAAmL;QACrL,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uEAAuE;iBACrF;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;oBACf,WAAW,EACT,wJAAwJ;iBAC3J;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,kDAAkD;iBAChE;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,6CAA6C;iBAC3D;gBACD,iBAAiB,EAAE;oBACjB,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,gDAAgD;iBAC9D;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;KACF;IACD;QACE,IAAI,EAAE,yBAAyB;QAC/B,WAAW,EACT,gHAAgH;QAClH,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,yCAAyC;iBACvD;gBACD,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,sDAAsD;iBACpE;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qDAAqD;iBACnE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;oBACvB,OAAO,EAAE,MAAM;iBAChB;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC;SACjC;KACF;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EACT,2HAA2H;QAC7H,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qBAAqB;iBACnC;gBACD,KAAK,EAAE;oBACL,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE;oBACX,WAAW,EAAE,2CAA2C;iBACzD;gBACD,cAAc,EAAE;oBACd,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,sDAAsD;iBACpE;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;KACF;CACF,CAAC;AAEF,SAAgB,YAAY;IAC1B,MAAM,MAAM,GAAG,IAAI,iBAAM,CACvB;QACE,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE;YACZ,KAAK,EAAE,EAAE;SACV;KACF,CACF,CAAC;IAEF,MAAM,CAAC,iBAAiB,CAAC,iCAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,gCAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAEjD,IAAI,CAAC;YACH,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;gBACxB,MAAM,SAAS,GAAG,wBAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC5C,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,KAAK,GAAG,CAAC,KAAK,OAAO,GAAG,CAAC,QAAQ,8BAA8B,GAAG,CAAC,QAAQ,CAAC,WAAW,YAAY,GAAG,CAAC,QAAQ,CAAC,MAAM,aAAa,GAAG,CAAC,QAAQ,CAAC,YAAY,YAAY,GAAG,CAAC,QAAQ,CAAC,gBAAgB,aAAa,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK;yBACrP;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,yBAAyB,EAAE,CAAC;gBACvC,MAAM,SAAS,GAAG,4BAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAChD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;yBACxC;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBAC/B,MAAM,SAAS,GAAG,8BAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;gBAElD,MAAM,gBAAgB,GAAG,GAAG,CAAC,OAAO;qBACjC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBACnB,OAAO,QAAQ,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,KACvD,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,OAAO,IAC/E,EAAE,CAAC;gBACL,CAAC,CAAC;qBACD,IAAI,CAAC,WAAW,CAAC,CAAC;gBAErB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,0BAA0B,GAAG,CAAC,KAAK,QAAQ,gBAAgB,2BAA2B,GAAG,CAAC,sBAAsB,oBAAoB,GAAG,CAAC,KAAK,GAAG;yBACvJ;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,mBAAmB,IAAI,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE;qBACnE;iBACF;aACF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,cAAc;IAClC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,+BAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;AACxE,CAAC"}
@@ -0,0 +1,50 @@
1
+ import { z } from "zod";
2
+ export declare const ReadUrlSchema: z.ZodObject<{
3
+ url: z.ZodString;
4
+ stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
5
+ max_age: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
6
+ include_links: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
7
+ only_main_content: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
8
+ }, "strip", z.ZodTypeAny, {
9
+ url: string;
10
+ stealth: "auto" | "force";
11
+ max_age: number;
12
+ include_links: boolean;
13
+ only_main_content: boolean;
14
+ }, {
15
+ url: string;
16
+ stealth?: "auto" | "force" | undefined;
17
+ max_age?: number | undefined;
18
+ include_links?: boolean | undefined;
19
+ only_main_content?: boolean | undefined;
20
+ }>;
21
+ export declare const ExtractDataSchema: z.ZodObject<{
22
+ url: z.ZodString;
23
+ json_schema: z.ZodRecord<z.ZodString, z.ZodAny>;
24
+ prompt: z.ZodOptional<z.ZodString>;
25
+ stealth: z.ZodDefault<z.ZodOptional<z.ZodEnum<["auto", "force"]>>>;
26
+ }, "strip", z.ZodTypeAny, {
27
+ url: string;
28
+ stealth: "auto" | "force";
29
+ json_schema: Record<string, any>;
30
+ prompt?: string | undefined;
31
+ }, {
32
+ url: string;
33
+ json_schema: Record<string, any>;
34
+ stealth?: "auto" | "force" | undefined;
35
+ prompt?: string | undefined;
36
+ }>;
37
+ export declare const SearchAndReadSchema: z.ZodObject<{
38
+ query: z.ZodString;
39
+ limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
40
+ scrape_results: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
41
+ }, "strip", z.ZodTypeAny, {
42
+ query: string;
43
+ limit: number;
44
+ scrape_results: boolean;
45
+ }, {
46
+ query: string;
47
+ limit?: number | undefined;
48
+ scrape_results?: boolean | undefined;
49
+ }>;
50
+ //# sourceMappingURL=tools.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;EA0BxB,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;EAc5B,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;EAe9B,CAAC"}
package/dist/tools.js ADDED
@@ -0,0 +1,61 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SearchAndReadSchema = exports.ExtractDataSchema = exports.ReadUrlSchema = void 0;
4
+ const zod_1 = require("zod");
5
+ exports.ReadUrlSchema = zod_1.z.object({
6
+ url: zod_1.z.string().url().describe("The full web URL to scrape and read into clean Markdown"),
7
+ stealth: zod_1.z
8
+ .enum(["auto", "force"])
9
+ .optional()
10
+ .default("auto")
11
+ .describe("'auto' tries fast TLS spoofing first and auto-escalates on 403/Turnstile; 'force' routes through full residential proxy + Turnstile solver immediately."),
12
+ max_age: zod_1.z
13
+ .number()
14
+ .int()
15
+ .nonnegative()
16
+ .optional()
17
+ .default(3600)
18
+ .describe("Redis cache TTL in seconds. Use 0 to force a real-time fresh bypass fetch."),
19
+ include_links: zod_1.z
20
+ .boolean()
21
+ .optional()
22
+ .default(true)
23
+ .describe("Whether to include extracted outgoing links in the response"),
24
+ only_main_content: zod_1.z
25
+ .boolean()
26
+ .optional()
27
+ .default(true)
28
+ .describe("Whether to strip navigational headers, footers, and ad noise using Trafilatura"),
29
+ });
30
+ exports.ExtractDataSchema = zod_1.z.object({
31
+ url: zod_1.z.string().url().describe("Target webpage URL to extract data from"),
32
+ json_schema: zod_1.z
33
+ .record(zod_1.z.any())
34
+ .describe("JSON Schema definition defining the desired typed structure"),
35
+ prompt: zod_1.z
36
+ .string()
37
+ .optional()
38
+ .describe("Optional extraction prompt or instructions for the LLM"),
39
+ stealth: zod_1.z
40
+ .enum(["auto", "force"])
41
+ .optional()
42
+ .default("auto")
43
+ .describe("Anti-bot bypass level"),
44
+ });
45
+ exports.SearchAndReadSchema = zod_1.z.object({
46
+ query: zod_1.z.string().min(1).describe("Search query to execute"),
47
+ limit: zod_1.z
48
+ .number()
49
+ .int()
50
+ .min(1)
51
+ .max(10)
52
+ .optional()
53
+ .default(5)
54
+ .describe("Number of top results to return (1-10)"),
55
+ scrape_results: zod_1.z
56
+ .boolean()
57
+ .optional()
58
+ .default(true)
59
+ .describe("Whether to concurrently fetch and convert each search result into clean Markdown"),
60
+ });
61
+ //# sourceMappingURL=tools.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAEX,QAAA,aAAa,GAAG,OAAC,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yDAAyD,CAAC;IACzF,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CACP,yJAAyJ,CAC1J;IACH,OAAO,EAAE,OAAC;SACP,MAAM,EAAE;SACR,GAAG,EAAE;SACL,WAAW,EAAE;SACb,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,aAAa,EAAE,OAAC;SACb,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,iBAAiB,EAAE,OAAC;SACjB,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gFAAgF,CAAC;CAC9F,CAAC,CAAC;AAEU,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACzE,WAAW,EAAE,OAAC;SACX,MAAM,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC;SACf,QAAQ,CAAC,6DAA6D,CAAC;IAC1E,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,wDAAwD,CAAC;IACrE,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACvB,QAAQ,EAAE;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,uBAAuB,CAAC;CACrC,CAAC,CAAC;AAEU,QAAA,mBAAmB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC1C,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC5D,KAAK,EAAE,OAAC;SACL,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CAAC,wCAAwC,CAAC;IACrD,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,48 @@
1
1
  {
2
2
  "name": "clearfetch",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "1.0.0",
4
+ "description": "Production-ready Model Context Protocol (MCP) server for ClearFetch (clearfetch.dev) - bypass Cloudflare, DataDome, and Turnstile to deliver clean Markdown & structured data to AI agents.",
5
+ "main": "dist/index.js",
6
+ "types": "dist/index.d.ts",
7
+ "bin": {
8
+ "clearfetch": "./bin/unblocked-mcp.js",
9
+ "clearfetch-mcp": "./bin/unblocked-mcp.js"
10
+ },
11
+ "scripts": {
12
+ "build": "tsc",
13
+ "start": "node ./bin/unblocked-mcp.js",
14
+ "dev": "tsc -w",
15
+ "prepublishOnly": "npm run build"
16
+ },
17
+ "keywords": [
18
+ "mcp",
19
+ "model-context-protocol",
20
+ "claude",
21
+ "cursor",
22
+ "windsurf",
23
+ "n8n",
24
+ "n8n-mcp",
25
+ "web-scraper",
26
+ "markdown",
27
+ "cloudflare-turnstile",
28
+ "datadome-bypass",
29
+ "ai-agent",
30
+ "rag"
31
+ ],
32
+ "author": "ClearFetch Team <support@clearfetch.dev>",
33
+ "license": "MIT",
34
+ "dependencies": {
35
+ "@modelcontextprotocol/sdk": "^1.6.0",
36
+ "dotenv": "^16.4.7",
37
+ "zod": "^3.24.2"
38
+ },
39
+ "devDependencies": {
40
+ "@types/node": "^22.13.9",
41
+ "typescript": "^5.8.2"
42
+ },
43
+ "files": [
44
+ "dist",
45
+ "bin",
46
+ "README.md"
47
+ ]
48
+ }