commoncrawl-mcp 1.0.0

package/README.md

@@ -0,0 +1,43 @@
+# commoncrawl-mcp
+
+Discover historical web captures through the Common Crawl index. This server is for provenance research, archival discovery, broken-link investigations, and finding the capture metadata needed for a later archive fetch.
+
+## Tools
+
+- `list_indexes`: list recent crawl collections.
+- `latest_index`: return the newest collection id.
+- `search_captures`: search URL or wildcard patterns and return capture index metadata such as timestamp, status, digest, filename, offset, and length.
+
+This server searches the index only. It does not download page contents, bypass access controls, or guarantee that a capture is complete or legally reusable. Requests and result pages are bounded.
+
+## Run
+
+```bash
+npm install
+npm run build
+node dist/index.js
+```
+
+## Quick start
+
+```bash
+npm install
+npm run build
+node dist/index.js
+```
+
+The server uses stdio, so it can be connected to Claude Desktop, Cursor, VS Code, MCP Inspector, or another compatible MCP client.
+
+## Tools at a glance
+
+- `list_indexes`: List available Common Crawl web crawl index collections, newest first.
+- `latest_index`: Return the identifier of the newest Common Crawl index collection.
+- `search_captures`: Find historical Common Crawl captures matching a URL or wildcard pattern. This returns index metadata, not page contents.
+
+## Limits and privacy
+
+This project is intentionally narrow. It should be treated as a practical helper, not a complete certification or security audit. Check the implementation and the returned data before using it with sensitive material. No credentials are required unless the project explicitly says otherwise.
+
+## Try it
+
+After building, connect the server through your MCP client. The repository root also contains `smoke-test.mjs` for projects covered by the shared harness. A typical tool call starts with `list_indexes`.

package/dist/api.d.ts

@@ -0,0 +1,18 @@
+export declare class CommonCrawlError extends Error {
+}
+type Collection = {
+    id?: string;
+    name?: string;
+    timegate?: string;
+    cd?: string;
+};
+export declare function listCollections(): Promise<Collection[]>;
+export declare function latestIndex(): Promise<string>;
+export declare function searchCaptures(urlPattern: string, index: string | undefined, page: number, limit: number): Promise<{
+    index: string;
+    page: number;
+    count: number;
+    records: any[];
+}>;
+export declare function format(value: unknown): string;
+export {};

package/dist/api.js

@@ -0,0 +1,44 @@
+const COLLECTIONS = "https://index.commoncrawl.org/collinfo.json";
+const HEADERS = { "User-Agent": "mrfentmen-commoncrawl-mcp/1.0" };
+export class CommonCrawlError extends Error {
+}
+async function getJson(url, timeout = 30000) {
+    const response = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(timeout) });
+    if (!response.ok)
+        throw new CommonCrawlError(`Common Crawl error ${response.status}`);
+    return response.json();
+}
+export async function listCollections() {
+    return (await getJson(new URL(COLLECTIONS)));
+}
+export async function latestIndex() {
+    const collections = await listCollections();
+    const id = collections[0]?.id;
+    if (!id)
+        throw new CommonCrawlError("Common Crawl returned no index collections");
+    return id;
+}
+export async function searchCaptures(urlPattern, index, page, limit) {
+    const selectedIndex = index || await latestIndex();
+    const url = new URL(`https://index.commoncrawl.org/${encodeURIComponent(selectedIndex)}-index`);
+    url.searchParams.set("url", urlPattern);
+    url.searchParams.set("output", "json");
+    url.searchParams.set("page", String(page));
+    url.searchParams.set("fl", "url,timestamp,status,mime,mime-detected,digest,filename,offset,length");
+    const response = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
+    if (!response.ok)
+        throw new CommonCrawlError(`Common Crawl index error ${response.status}`);
+    const body = await response.text();
+    const records = body.split("\\n").filter(Boolean).slice(0, limit).map((line) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch {
+            return { raw: line.slice(0, 1000) };
+        }
+    });
+    return { index: selectedIndex, page, count: records.length, records };
+}
+export function format(value) {
+    return JSON.stringify(value, null, 2).slice(0, 16000);
+}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.js

@@ -0,0 +1,12 @@
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { createServer } from "./server.js";
+async function main() {
+    const server = createServer();
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error("MCP server running on stdio");
+}
+main().catch((err) => {
+    console.error("Fatal error:", err);
+    process.exit(1);
+});

package/dist/server.d.ts

@@ -0,0 +1,2 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+export declare function createServer(): McpServer;

package/dist/server.js

@@ -0,0 +1,55 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { format, latestIndex, listCollections, searchCaptures } from "./api.js";
+const text = (value) => ({ content: [{ type: "text", text: value }] });
+const textError = (t) => ({ content: [{ type: "text", text: t }], isError: true });
+const READ_ONLY = { readOnlyHint: true, openWorldHint: true };
+const errorText = (error) => text(`Error: ${error instanceof Error ? error.message : String(error)}`);
+export function createServer() {
+    const server = new McpServer({ name: "commoncrawl-mcp", version: "1.0.0" });
+    server.registerTool("list_indexes", {
+        title: "List indexes",
+        description: "List available Common Crawl web crawl index collections, newest first.",
+        inputSchema: z.object({}),
+        annotations: READ_ONLY,
+    }, async () => {
+        try {
+            return text(format((await listCollections()).slice(0, 30)));
+        }
+        catch (error) {
+            return errorText(error);
+        }
+    });
+    server.registerTool("latest_index", {
+        title: "Latest index",
+        description: "Return the identifier of the newest Common Crawl index collection.",
+        inputSchema: z.object({}),
+        annotations: READ_ONLY,
+    }, async () => {
+        try {
+            return text(await latestIndex());
+        }
+        catch (error) {
+            return errorText(error);
+        }
+    });
+    server.registerTool("search_captures", {
+        title: "Search captures",
+        description: "Find historical Common Crawl captures matching a URL or wildcard pattern. This returns index metadata, not page contents.",
+        inputSchema: z.object({
+            url: z.string().min(1).max(500).describe("URL or wildcard pattern, for example example.com/*"),
+            index: z.string().regex(/^[A-Za-z0-9._-]+$/).optional().describe("Optional collection id such as CC-MAIN-2025-30"),
+            page: z.number().int().min(1).max(20).default(1),
+            limit: z.number().int().min(1).max(100).default(20),
+        }),
+        annotations: READ_ONLY,
+    }, async ({ url, index, page, limit }) => {
+        try {
+            return text(format(await searchCaptures(url, index, page, limit)));
+        }
+        catch (error) {
+            return errorText(error);
+        }
+    });
+    return server;
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "commoncrawl-mcp",
+  "version": "1.0.0",
+  "description": "Use this MCP server to Common Crawl index discovery for historical web captures. Tools include list indexes, latest index, search captures",
+  "type": "module",
+  "mcpName": "io.github.mrfentmen/commoncrawl-mcp",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mrfentmen/commoncrawl-mcp.git"
+  },
+  "bin": {
+    "commoncrawl-mcp": "./dist/index.js"
+  },
+  "main": "./dist/index.js",
+  "files": [
+    "dist",
+    "server.json",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "start": "node dist/index.js",
+    "dev": "npm run build && node dist/index.js"
+  },
+  "keywords": [
+    "mcp",
+    "public-data",
+    "commoncrawl"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.6.0"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}

package/server.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.mrfentmen/commoncrawl-mcp",
+  "description": "Use this MCP server to Common Crawl index discovery for historical web captures. Tools include...",
+  "repository": {
+    "url": "https://github.com/mrfentmen/commoncrawl-mcp",
+    "source": "github"
+  },
+  "version": "1.0.0",
+  "packages": [
+    {
+      "registryType": "npm",
+      "identifier": "commoncrawl-mcp",
+      "version": "1.0.0",
+      "transport": {
+        "type": "stdio"
+      }
+    }
+  ]
+}