@blazecrawl/mcp 0.1.2

package/README.md

@@ -0,0 +1,30 @@
+# @blazecrawl/mcp
+
+MCP server exposing a self-hosted BlazeCrawl Core instance's `scrape`, `map`,
+and `crawl` tools over stdio. Requires Node.js 18+.
+
+## Install
+
+```bash
+npm install @blazecrawl/mcp@0.1.2
+```
+
+## Generic MCP client configuration
+
+```json
+{
+  "mcpServers": {
+    "blazecrawl": {
+      "command": "npx",
+      "args": ["-y", "@blazecrawl/mcp@0.1.2"],
+      "env": {
+        "BLAZECRAWL_API_URL": "http://127.0.0.1:8000",
+        "BLAZECRAWL_API_KEY": "blz_local_..."
+      }
+    }
+  }
+}
+```
+
+The server forwards tool calls to BlazeCrawl. Crawl operations return a job
+reference; use the crawl-status tool to poll it.

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@blazecrawl/mcp",
+  "version": "0.1.2",
+  "description": "MCP server for BlazeCrawl Core (scrape/map/crawl tools).",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/danishxsethi/blazecrawl.git"
+  },
+  "homepage": "https://github.com/danishxsethi/blazecrawl#readme",
+  "bugs": {
+    "url": "https://github.com/danishxsethi/blazecrawl/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "type": "module",
+  "main": "./src/index.js",
+  "bin": { "blazecrawl-mcp": "./src/index.js" },
+  "engines": { "node": ">=18" },
+  "files": ["src"]
+}

package/src/index.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/**
+ * BlazeCrawl MCP server (Node.js) — scrape / map / crawl tools over stdio.
+ *
+ * Implements the MCP stdio JSON-RPC handshake directly (no SDK dependency).
+ *
+ *   BLAZECRAWL_API_URL   (default http://127.0.0.1:8000)
+ *   BLAZECRAWL_API_KEY
+ */
+
+const API_KEY = process.env.BLAZECRAWL_API_KEY || null;
+const BASE = (process.env.BLAZECRAWL_API_URL || "http://127.0.0.1:8000").replace(/\/+$/, "");
+
+const TOOLS = [
+  {
+    name: "scrape",
+    description: "Scrape a URL and return clean Markdown (LLM-ready).",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url: { type: "string" },
+        render: { type: "string", enum: ["auto", "static", "browser"], default: "auto" },
+      },
+      required: ["url"],
+    },
+  },
+  {
+    name: "map",
+    description: "Discover the URL set of a site (sitemap + link graph).",
+    inputSchema: { type: "object", properties: { url: { type: "string" } }, required: ["url"] },
+  },
+  {
+    name: "crawl",
+    description: "Crawl a site (BFS, same-origin) and return page Markdown.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url: { type: "string" },
+        max_pages: { type: "integer" },
+        max_depth: { type: "integer" },
+      },
+      required: ["url"],
+    },
+  },
+];
+
+function headers() {
+  const h = { "Content-Type": "application/json" };
+  if (API_KEY) h["Authorization"] = `Bearer ${API_KEY}`;
+  return h;
+}
+
+async function post(path, body) {
+  const r = await fetch(`${BASE}${path}`, { method: "POST", headers: headers(), body: JSON.stringify(body) });
+  return r.json();
+}
+async function get(path) {
+  const r = await fetch(`${BASE}${path}`, { headers: headers() });
+  return r.json();
+}
+
+async function callTool(name, args) {
+  if (name === "scrape") {
+    const body = { url: args.url };
+    if (args.render) body.render = args.render;
+    const d = await post("/v1/scrape", body);
+    return (d.data && d.data.markdown) || JSON.stringify(d);
+  }
+  if (name === "map") {
+    const d = await post("/v1/map", { url: args.url });
+    return (d.urls || []).join("\n") || JSON.stringify(d);
+  }
+  if (name === "crawl") {
+    const body = { url: args.url };
+    if (args.max_pages) body.max_pages = args.max_pages;
+    if (args.max_depth !== undefined) body.max_depth = args.max_depth;
+    const job = await post("/v1/crawl", body);
+    const id = job.job_id;
+    for (let i = 0; i < 600; i++) {
+      const st = await get(`/v1/crawl/${id}`);
+      if (["completed", "failed", "cancelled"].includes(st.status)) {
+        return (st.pages || []).map((p) => `## ${p.url}\n\n${p.markdown || ""}`).join("\n\n") || JSON.stringify(st);
+      }
+      await new Promise((r) => setTimeout(r, 1000));
+    }
+    return "crawl timed out";
+  }
+  return `unknown tool: ${name}`;
+}
+
+function send(msg) {
+  process.stdout.write(JSON.stringify(msg) + "\n");
+}
+
+let buffer = "";
+process.stdin.on("data", (chunk) => {
+  buffer += chunk;
+  let idx;
+  while ((idx = buffer.indexOf("\n")) >= 0) {
+    const line = buffer.slice(0, idx).trim();
+    buffer = buffer.slice(idx + 1);
+    if (!line) continue;
+    let msg;
+    try {
+      msg = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    handle(msg).catch(() => {});
+  }
+});
+
+async function handle(msg) {
+  const { id, method, params } = msg;
+  if (method === "initialize") {
+    send({
+      jsonrpc: "2.0",
+      id,
+      result: {
+        protocolVersion: "2024-11-05",
+        capabilities: { tools: {} },
+        serverInfo: { name: "blazecrawl", version: "0.1.0" },
+      },
+    });
+    return;
+  }
+  if (method === "notifications/initialized") return;
+  if (method === "tools/list") {
+    send({ jsonrpc: "2.0", id, result: { tools: TOOLS } });
+    return;
+  }
+  if (method === "tools/call") {
+    try {
+      const text = await callTool(params.name, params.arguments || {});
+      send({ jsonrpc: "2.0", id, result: { content: [{ type: "text", text }] } });
+    } catch (e) {
+      send({ jsonrpc: "2.0", id, result: { content: [{ type: "text", text: String(e) }], isError: true } });
+    }
+    return;
+  }
+  if (id !== undefined) {
+    send({ jsonrpc: "2.0", id, error: { code: -32601, message: `method not found: ${method}` } });
+  }
+}