distil-mcp 1.2.0

package/LICENSE.md

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 exec.io pty ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

package/README.md

@@ -0,0 +1,85 @@
+# @distil.net/mcp
+
+MCP for the [Distil](https://Distil.ai) web proxy. Provides a fall through CLI and also Gives Claude Desktop, Cursor, Windsurf and any MCP-compatible AI tool the ability to fetch web pages and search the web — clean Markdown, no noise.
+
+## Tools
+
+| Tool | Description |
+|------|-------------|
+| `distil_scrape` | Fetch a URL and return its content as clean Markdown. Handles JavaScript-rendered pages, PDFs, and automatic content extraction. |
+| `distil_search` | Search the web and return results as Markdown. Includes titles, URLs, and snippet text for the top results. |
+| `distil_screenshot` | Take a screenshot of a web page and return it as an image. |
+| `distil_render` | Render a web page (such as a single page javascript app) before trying to extract markdown. |
+| `distil_raw` | Fetch a URL and return its raw content bypassing any attempt to render markdown. |
+| `distil_nocache` | Fetch a URL and return its content without using the cache. |
+
+
+## Setup
+
+### 1. Get a Distil API key
+
+Sign up at [distil.ai](https://distil.ai) to get your API key (`dk_...`).
+
+### 2. Add to your MCP config
+
+**Claude Desktop** (`~/Library/Application Support/Claude/claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "Distil": {
+      "command": "npx",
+      "args": ["-y", "@distil.net/mcp"],
+      "env": {
+        "DISTIL_API_KEY": "dk_your_key_here"
+      }
+    }
+  }
+}
+```
+
+**Cursor / Windsurf** — add the same block to your MCP settings file.
+
+### 3. Restart your AI tool
+
+The `distil_scrape`, `distil_search`, `distil_screenshot`, `distil_render`, `distil_raw` and `distil_nocache` tools will appear automatically.
+
+## Configuration
+
+| Environment Variable | Default | Description |
+|---------------------|---------|-------------|
+| `DISTIL_API_KEY` | *(required)* | Your Distil API key |
+| `DISTIL_PROXY_URL` | `https://proxy.distil.ai` | Proxy base URL (override for self-hosted) |
+
+## Usage examples
+
+Once installed, you can ask your AI assistant things like:
+
+- *"Fetch https://example.com and summarise it"* → uses `distil_scrape`
+- *"Search the web for the latest Node.js release notes"* → uses `distil_search`
+- *"Take a screen shot of npmjs.org"* → uses `distil_screenshot`
+- *"Go and read the web page of openai.com"* → uses `distil_render`
+- *"Make sure you get the latest version of openclaw.ai"* → uses `distil_nocache`
+
+## Running manually
+
+```bash
+DISTIL_API_KEY=dk_your_key npx @distil.net/mcp
+```
+
+The server speaks [MCP JSON-RPC 2.0](https://modelcontextprotocol.io) over stdio.
+
+## Development
+
+```bash
+# Clone the Distil repo
+git clone https://github.com/exec-io/distil-mcp.git
+cd distil.net/mcp
+
+# Run tests (no dependencies needed — uses Node built-ins only)
+node --test test.js
+```
+
+## License
+
+MIT — see [LICENSE.md](../LICENSE.md) in the repo root.

package/index.js

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+// distil-mcp — MCP server for the Distil proxy
+// Exposes distil_scrape, distil_search, distil_screenshot, distil_render, distil_raw
+// and distil_nocache as MCP tools over JSON-RPC 2.0 stdio.
+// Usage: npx distil-mcp (set DISTIL_API_KEY env var first)
+'use strict';
+
+const https = require('https');
+const http = require('http');
+const { URL } = require('url');
+
+const PROXY_BASE = (process.env.DISTIL_PROXY_URL || 'https://proxy.Distil.ai').replace(/\/$/, '');
+let   API_KEY    = process.env.DISTIL_API_KEY || '';
+const VERSION    = require('./package.json').version;
+
+// ── CLI mode ──────────────────────────────────────────────────────────────────
+// If invoked with a subcommand (fetch/search), run as one-shot CLI and exit.
+const [,, cmd, ...cliArgs] = process.argv;
+
+if (cmd === 'fetch' || cmd === 'search') {
+  if (!API_KEY) {
+    // Try macOS Keychain fallback
+    const { execSync } = require('child_process');
+    try {
+      process.env.DISTIL_API_KEY = execSync(
+        'security find-generic-password -s distil-api-key -w 2>/dev/null',
+        { encoding: 'utf8', stdio: ['pipe','pipe','pipe'] }
+      ).trim();
+      API_KEY = process.env.DISTIL_API_KEY;
+    } catch (_) {}
+  }
+
+  (async () => {
+    try {
+      if (cmd === 'fetch') {
+        if (!cliArgs[0]) { process.stderr.write('Usage: distil fetch <url>\n'); process.exit(1); }
+        const result = await callTool('distil_scrape', { url: cliArgs[0] });
+        process.stdout.write(result + '\n');
+      } else {
+        if (!cliArgs.length) { process.stderr.write('Usage: distil search <query>\n'); process.exit(1); }
+        const result = await callTool('distil_search', { query: cliArgs.join(' ') });
+        process.stdout.write(result + '\n');
+      }
+    } catch (e) {
+      process.stderr.write('Error: ' + e.message + '\n');
+      process.exit(1);
+    }
+  })();
+} else {
+  // No CLI subcommand — fall through to MCP stdio server below
+  startMCPServer();
+}
+
+// ── HTTP helper ───────────────────────────────────────────────────────────────
+
+function get(url, headers) {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const mod = parsed.protocol === 'https:' ? https : http;
+    const req = mod.get(url, { headers }, (res) => {
+      if (res.statusCode === 301 || res.statusCode === 302) {
+        return get(res.headers.location, headers).then(resolve, reject);
+      }
+      const chunks = [];
+      res.on('data', (c) => chunks.push(c));
+      res.on('end',  () => resolve(Buffer.concat(chunks).toString('utf8')));
+    });
+    req.on('error', reject);
+    req.setTimeout(30000, () => req.destroy(new Error('Request timed out')));
+  });
+}
+
+// ── Tool definitions ──────────────────────────────────────────────────────────
+
+const TOOLS = [
+  {
+    name: 'distil_scrape',
+    description: 'Fetch a URL and return its content as clean Markdown. Handles JavaScript-rendered pages, PDFs, and automatic content extraction.',
+    inputSchema: {
+      type: 'object',
+      properties: { url: { type: 'string', description: 'The URL to fetch and convert to Markdown' } },
+      required: ['url'],
+    },
+  },
+  {
+    name: 'distil_search',
+    description: 'Search the web and return results as Markdown. Includes titles, URLs, and snippet text for the top results.',
+    inputSchema: {
+      type: 'object',
+      properties: { query: { type: 'string', description: 'The search query' } },
+      required: ['query'],
+    },
+  },
+  {
+    name: 'distil_screenshot',
+    description: 'Take a screenshot of a web page and return it as an image.',
+    inputSchema: {
+      type: 'object',
+      properties: { url: { type: 'string', description: 'The URL of the web page to screenshot' } },
+      required: ['url'],
+    },
+  },
+  {
+    name: 'distil_render',
+    description: 'Render a javascript single page application (SPA) before trying to extract markdown.',
+    inputSchema: {
+      type: 'object',
+      properties: { url: { type: 'string', description: 'The URL of the javascript SPA to render' } },
+      required: ['url'],
+    },
+  },
+  {
+    name: 'distil_raw',
+    description: 'Fetch a URL and return its raw content bypassing any attempt to render markdown.',
+    inputSchema: {
+      type: 'object',
+      properties: { url: { type: 'string', description: 'The URL to fetch' } },
+      required: ['url'],
+    },
+  },
+  {
+    name: 'distil_nocache',
+    description: 'Fetch a URL and return its content without using the cache.',
+    inputSchema: {
+      type: 'object',
+      properties: { url: { type: 'string', description: 'The URL to fetch uncached' } },
+      required: ['url'],
+    },
+  },
+];
+
+// ── Tool execution ────────────────────────────────────────────────────────────
+
+async function callTool(name, args) {
+  if (!API_KEY) throw new Error('DISTIL_API_KEY environment variable is required');
+  const headers = { 'X-Distil-Key': API_KEY };
+
+  if (name === 'distil_scrape') {
+    if (!args.url) throw new Error('url is required');
+    return get(`${PROXY_BASE}/${args.url}`, headers);
+  }
+  if (name === 'distil_search') {
+    if (!args.query) throw new Error('query is required');
+    return get(`${PROXY_BASE}/search?q=${encodeURIComponent(args.query)}`, { ...headers, 'Accept': 'text/markdown' });
+  }
+  if (name === 'distil_screenshot') {
+    if (!args.url) throw new Error('url is required');
+    return get(`${PROXY_BASE}/screenshot/${args.url}`, headers);
+  }
+  if (name === 'distil_render') {
+    if (!args.url) throw new Error('url is required');
+    return get(`${PROXY_BASE}/render/${args.url}`, headers);
+  }
+  if (name === 'distil_raw') {
+    if (!args.url) throw new Error('url is required');
+    return get(`${PROXY_BASE}/raw/${args.url}`, headers);
+  }
+  if (name === 'distil_nocache') {
+    if (!args.url) throw new Error('url is required');
+    return get(`${PROXY_BASE}/nocache/${args.url}`, headers);
+  }
+  throw new Error(`Unknown tool: ${name}`);
+}
+
+// ── JSON-RPC transport ────────────────────────────────────────────────────────
+
+const write = (obj) => process.stdout.write(JSON.stringify(obj) + '\n');
+const ok    = (id, result) => write({ jsonrpc: '2.0', id, result });
+const fail  = (id, code, message) => write({ jsonrpc: '2.0', id, error: { code, message } });
+
+async function dispatch(msg) {
+  const { id, method, params } = msg;
+  try {
+    switch (method) {
+      case 'initialize':
+        ok(id, { protocolVersion: '2024-11-05', capabilities: { tools: {} }, serverInfo: { name: '@distil.net/mcp', version: VERSION } });
+        break;
+      case 'initialized':
+      case 'notifications/initialized':
+        break; // notification — no response
+      case 'ping':
+        ok(id, {});
+        break;
+      case 'tools/list':
+        ok(id, { tools: TOOLS });
+        break;
+      case 'tools/call': {
+        const text = await callTool(params.name, params.arguments || {});
+        ok(id, { content: [{ type: 'text', text }] });
+        break;
+      }
+      default:
+        if (id != null) fail(id, -32601, `Method not found: ${method}`);
+    }
+  } catch (err) {
+    if (id != null) fail(id, -32603, err.message);
+  }
+}
+
+// ── Stdin reader ──────────────────────────────────────────────────────────────
+
+function startMCPServer() {
+  let buf = '';
+  process.stdin.setEncoding('utf8');
+  process.stdin.on('data', (chunk) => {
+    buf += chunk;
+    const lines = buf.split('\n');
+    buf = lines.pop();
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try { dispatch(JSON.parse(line)); }
+      catch { fail(null, -32700, 'Parse error'); }
+    }
+  });
+  process.stdin.on('end', () => process.exit(0));
+}

package/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "distil-mcp",
+  "version": "1.2.0",
+  "description": "MCP server for the Distil web proxy — distil_scrape, distil_search, distil_screenshot, distil_raw, distil_render, distil_nocache for Claude Desktop, Cursor, Windsurf, and any MCP-compatible AI tool",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/exec-io/distil-mcp.git",
+    "directory": "mcp"
+  },
+  "keywords": ["mcp", "distil", "web-scraping", "search", "ai", "claude", "cursor", "openclaw"],
+  "engines": { "node": ">=18" },
+  "bin": {
+    "distil": "./index.js"
+  },
+  "scripts": {
+    "test": "node --test test.js",
+    "start": "node index.js"
+  },
+  "files": ["index.js", "README.md"]
+}