commoncrawl-mcp 1.0.1 → 1.0.2

package/README.md

@@ -41,3 +41,22 @@ This project is intentionally narrow. It should be treated as a practical helper
 ## Try it
 
 After building, connect the server through your MCP client. The repository root also contains `smoke-test.mjs` for projects covered by the shared harness. A typical tool call starts with `list_indexes`.
+
+<!-- paywall -->
+## Premium tools
+
+These tools need a license key:
+
+* `capture_history`
+* `index_stats`
+
+Buy a key at https://mcp-marketplace.io/server/io-github-mrfentmen-commoncrawl-mcp and set it in your MCP client config:
+
+```json
+"env": { "MCP_LICENSE_KEY": "mcp_live_..." }
+```
+
+The key is checked against MCP Marketplace, cached for 24 hours, and keeps
+working offline once one check has succeeded. Every other tool on this server
+stays free.
+<!-- /paywall -->

package/dist/api.d.ts

@@ -15,4 +15,24 @@ export declare function searchCaptures(urlPattern: string, index: string | undef
     records: any[];
 }>;
 export declare function format(value: unknown): string;
+export interface HistoryArgs {
+    url: string;
+    index?: string;
+    match_type?: string;
+    from_year?: number;
+    to_year?: number;
+    limit?: number;
+}
+export declare function captureHistory(args: HistoryArgs): Promise<string>;
+export interface StatsArgs {
+    url: string;
+    index?: string;
+    match_type?: string;
+    sample?: number;
+}
+export declare function indexStats(args: StatsArgs): Promise<string>;
+export interface CollectionArgs {
+    id: string;
+}
+export declare function collectionDetails(args: CollectionArgs): Promise<string>;
 export {};

package/dist/api.js

@@ -42,3 +42,170 @@ export async function searchCaptures(urlPattern, index, page, limit) {
 export function format(value) {
     return JSON.stringify(value, null, 2).slice(0, 16000);
 }
+// ---- muxA tools:
+const CC = "https://index.commoncrawl.org";
+function cap(value, fallback, max) {
+    return Math.max(1, Math.min(Math.round(value ?? fallback), max));
+}
+function ccLine(raw) {
+    try {
+        const d = JSON.parse(raw);
+        return typeof d?.url === "string" ? d : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function cdx(index, params, limit) {
+    const query = new URLSearchParams({ ...params, output: "json" });
+    const url = `${CC}/${index}-index?${query.toString()}`;
+    const res = await fetch(url, {
+        headers: { "User-Agent": "mrfentmen-commoncrawl-mcp/1.0", Accept: "application/json" },
+        signal: AbortSignal.timeout(45000),
+    });
+    if (!res.ok) {
+        const body = await res.text().catch(() => "");
+        throw new Error(`Common Crawl index returned ${res.status}. ${body.slice(0, 200)}`.trim());
+    }
+    const text = await res.text();
+    const rows = [];
+    for (const line of text.split("\n")) {
+        if (!line.trim())
+            continue;
+        const d = ccLine(line);
+        if (d)
+            rows.push(d);
+        if (rows.length >= limit)
+            break;
+    }
+    return { rows, pagesFetched: 1 };
+}
+async function pickIndex(id) {
+    if (id?.trim())
+        return id.trim();
+    return latestIndex();
+}
+function stamp(value) {
+    const s = String(value ?? "");
+    return /^\d{14}$/.test(s) ? `${s.slice(0, 4)}-${s.slice(4, 6)}-${s.slice(6, 8)} ${s.slice(8, 10)}:${s.slice(10, 12)}:${s.slice(12, 14)} UTC` : s || "unknown";
+}
+export async function captureHistory(args) {
+    const target = (args.url ?? "").trim();
+    if (!target)
+        throw new Error("Provide a URL or domain to trace, e.g. example.com or example.com/docs/.");
+    const matchType = (args.match_type ?? "prefix").trim().toLowerCase();
+    if (!["exact", "prefix", "host", "domain"].includes(matchType)) {
+        return `Unknown match_type "${args.match_type}". Use exact, prefix, host or domain.`;
+    }
+    const index = await pickIndex(args.index);
+    const params = { url: target, matchType };
+    const from = Number(args.from_year);
+    const to = Number(args.to_year);
+    if (Number.isFinite(from) && from > 1900)
+        params.from = String(Math.round(from));
+    if (Number.isFinite(to) && to > 1900)
+        params.to = String(Math.round(to));
+    const limit = cap(args.limit, 20, 200);
+    const { rows } = await cdx(index, params, limit);
+    if (!rows.length) {
+        return `Common Crawl ${index} holds no ${matchType} capture of ${target}${params.from ? ` from ${params.from}` : ""}${params.to ? ` to ${params.to}` : ""}. The URL may not have been crawled in that crawl.`;
+    }
+    const times = rows.map((r) => Number(r.timestamp) || 0).filter((t) => t > 0);
+    const statuses = new Map();
+    const mimes = new Map();
+    for (const r of rows) {
+        const st = String(r.status ?? "?");
+        statuses.set(st, (statuses.get(st) ?? 0) + 1);
+        const m = String(r["mime-detected"] ?? r.mime ?? "?");
+        mimes.set(m, (mimes.get(m) ?? 0) + 1);
+    }
+    const distinct = new Set(rows.map((r) => r.url)).size;
+    return [
+        `Common Crawl ${index} history for ${target} (matchType ${matchType}): ${rows.length} capture record(s) covering ${distinct} distinct URL(s).`,
+        `Capture window ${times.length ? `${stamp(Math.min(...times))} to ${stamp(Math.max(...times))}` : "not derivable from the returned records"}`,
+        `Status codes: ${[...statuses.entries()].map(([k, v]) => `${k} ${v}`).join(", ")}`,
+        `Detected MIME types: ${[...mimes.entries()].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k} ${v}`).join(", ")}`,
+        "",
+        ...rows.slice(0, cap(args.limit, 20, 200)).map((r, i) => `${i + 1}. ${stamp(r.timestamp)} | ${r.status ?? "?"} ${r.mime ?? ""} | ${String(r.url).slice(0, 100)}` +
+            `${r.digest ? ` | digest ${String(r.digest).slice(0, 16)}` : ""}${r.length ? ` | ${r.length} bytes` : ""}`),
+        "The digest is a base32 SHA-1 of the captured payload, so you can tell whether two crawls saw identical content.",
+    ].join("\n");
+}
+export async function indexStats(args) {
+    const target = (args.url ?? "").trim();
+    if (!target)
+        throw new Error("Provide a URL or domain to profile, e.g. example.com.");
+    const matchType = (args.match_type ?? "prefix").trim().toLowerCase();
+    if (!["exact", "prefix", "host", "domain"].includes(matchType)) {
+        return `Unknown match_type "${args.match_type}". Use exact, prefix, host or domain.`;
+    }
+    const index = await pickIndex(args.index);
+    const sample = cap(args.sample, 500, 5000);
+    const { rows } = await cdx(index, { url: target, matchType }, sample);
+    if (!rows.length)
+        return `Common Crawl ${index} returned no ${matchType} records for ${target}.`;
+    const statuses = new Map();
+    const mimes = new Map();
+    const depths = new Map();
+    const schemes = new Map();
+    let bytes = 0;
+    let counted = 0;
+    for (const r of rows) {
+        statuses.set(String(r.status ?? "?"), (statuses.get(String(r.status ?? "?")) ?? 0) + 1);
+        mimes.set(String(r["mime-detected"] ?? r.mime ?? "?"), (mimes.get(String(r["mime-detected"] ?? r.mime ?? "?")) ?? 0) + 1);
+        schemes.set(String(r.url ?? "").split(":")[0] || "?", (schemes.get(String(r.url ?? "").split(":")[0] || "?") ?? 0) + 1);
+        try {
+            const u = new URL(String(r.url));
+            const depth = u.pathname.split("/").filter(Boolean).length;
+            depths.set(String(depth), (depths.get(String(depth)) ?? 0) + 1);
+        }
+        catch {
+            /* ignore unparsable urls */
+        }
+        const len = Number(r.length);
+        if (Number.isFinite(len) && len > 0) {
+            bytes += len;
+            counted += 1;
+        }
+    }
+    const ok = rows.filter((r) => Number(r.status) >= 200 && Number(r.status) < 300).length;
+    return [
+        `Common Crawl ${index} profile of ${target} (matchType ${matchType}), ${rows.length} record(s) sampled.`,
+        `Status codes: ${[...statuses.entries()].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k} ${v}`).join(", ")} | 2xx share ${((ok / rows.length) * 100).toFixed(1)}%`,
+        `MIME types: ${[...mimes.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([k, v]) => `${k} ${v}`).join(", ")}`,
+        `Schemes: ${[...schemes.entries()].map(([k, v]) => `${k} ${v}`).join(", ")}`,
+        `Path depth distribution: ${[...depths.entries()].sort((a, b) => Number(a[0]) - Number(b[0])).map(([k, v]) => `${k}:${v}`).join(" ")}`,
+        counted ? `Payload bytes seen across ${counted.toLocaleString("en-US")} record(s): ${(bytes / 1024 / 1024).toFixed(1)} MB, mean ${Math.round(bytes / counted).toLocaleString("en-US")} B` : "No record lengths reported in this sample",
+        rows.length >= sample ? `Sample was capped at ${sample} records, so these are proportions from that slice.` : "This is the complete record set the index returned for the query.",
+    ].join("\n");
+}
+function crawlWindow(hit) {
+    const raw = hit;
+    if (raw.from && raw.to)
+        return `Crawl window: ${String(raw.from)} to ${String(raw.to)}`;
+    return "Crawl window not reported";
+}
+function indexSize(hit) {
+    const raw = hit;
+    const value = raw["total-index-size"];
+    return value != null ? `${String(value)} bytes` : "not reported";
+}
+export async function collectionDetails(args) {
+    const id = (args.id ?? "").trim();
+    if (!id)
+        throw new Error("Provide a collection id, e.g. CC-MAIN-2026-39.");
+    const d = (await listCollections());
+    const hit = d.find((c) => c.id === id);
+    if (!hit) {
+        return `"${id}" is not in the Common Crawl collection list. The newest collections are ${d.slice(0, 5).map((c) => c.id).join(", ")}.`;
+    }
+    return [
+        `Common Crawl collection ${String(hit.id)}: ${String(hit.name ?? "unnamed")}`,
+        `Timegate: ${hit.timegate ?? "not reported"}`,
+        `CDX API: ${hit["cdx-api"] ?? "not reported"}`,
+        crawlWindow(hit),
+        `Total index size: ${indexSize(hit)}`,
+        `This collection is index number ${d.indexOf(hit) + 1} of ${d.length} listed.`,
+        "The index only holds metadata and offsets; the captured bytes live in WARC files on S3 and are fetched by range request using the filename and offset in each record.",
+    ].join("\n");
+}

package/dist/license.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Returns null when the caller may run the tool, or a short message telling
+ * them how to get a key.
+ *
+ * A good key is remembered for an hour, so a busy session does not hit the
+ * license server on every call and keeps working through a brief outage.
+ */
+export declare function premiumRequired(tool: string): Promise<string | null>;

package/dist/license.js

@@ -0,0 +1,79 @@
+// Premium tools on this server need a license key bought from MCP Marketplace.
+// Buyers put the key in their MCP client config as MCP_LICENSE_KEY. Every tool
+// that is not listed in PREMIUM keeps working without a key.
+const SLUG = "commoncrawl-mcp";
+// Tools that need a key. Everything else is free.
+const PREMIUM = new Set([
+    "capture_history",
+    "index_stats",
+]);
+const BUY_URL = `https://mcp-marketplace.io/server/io-github-mrfentmen-${SLUG}`;
+// The license check calls the marketplace's verify endpoint directly instead of
+// using @mcp_marketplace/license. That SDK sends no `apikey` header, so every
+// check it makes is rejected by Supabase before it reaches the function. The
+// publishable key below is public by design - it ships in mcp-marketplace.io's
+// own JavaScript and only ever reaches their licence check.
+const DEFAULT_VERIFY_URL = "https://virupvwhtkpkjsiskckg.supabase.co/functions/v1/verify-key";
+const PUBLISHABLE_KEY = "sb_publishable_BuqlW96Ke8C_zzJG-LQv1Q_YHi_r_4h";
+const OK_CACHE_MS = 60 * 60 * 1000;
+const BAD_CACHE_MS = 60 * 1000;
+const seen = new Map();
+const REASONS = {
+    missing_key: "no key is set",
+    invalid_format: "that key is not a valid MCP Marketplace key",
+    not_found: "that key was not found",
+    revoked: "that key has been revoked",
+    rotated: "that key was rotated, use your new one",
+    expired: "that key has expired, renew it",
+    rate_limited: "there were too many checks just now, try again shortly",
+    network_error: "the license server could not be reached",
+};
+function blocked(tool, reason) {
+    const why = REASONS[reason] ?? `the license server answered "${reason}"`;
+    return `"${tool}" needs a license key, but ${why}. ` +
+        `Set MCP_LICENSE_KEY in your MCP client config. Get a key: ${BUY_URL}`;
+}
+async function verify(key) {
+    const res = await fetch(process.env.MCP_LICENSE_VERIFY_URL || DEFAULT_VERIFY_URL, {
+        method: "POST",
+        headers: {
+            apikey: PUBLISHABLE_KEY,
+            Authorization: `Bearer ${PUBLISHABLE_KEY}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ key, slug: SLUG }),
+        // an unusable key answers 400 with a JSON body, so read the body either way
+        signal: AbortSignal.timeout(15000),
+    });
+    const data = (await res.json());
+    if (typeof data?.valid !== "boolean")
+        return { valid: false, reason: "unexpected_response" };
+    return data;
+}
+/**
+ * Returns null when the caller may run the tool, or a short message telling
+ * them how to get a key.
+ *
+ * A good key is remembered for an hour, so a busy session does not hit the
+ * license server on every call and keeps working through a brief outage.
+ */
+export async function premiumRequired(tool) {
+    if (!PREMIUM.has(tool))
+        return null;
+    const key = process.env.MCP_LICENSE_KEY;
+    if (!key)
+        return blocked(tool, "missing_key");
+    const hit = seen.get(key);
+    if (hit && Date.now() - hit.at < (hit.valid ? OK_CACHE_MS : BAD_CACHE_MS)) {
+        return hit.valid ? null : blocked(tool, hit.reason ?? "invalid");
+    }
+    let result;
+    try {
+        result = await verify(key);
+    }
+    catch {
+        result = { valid: false, reason: "network_error" };
+    }
+    seen.set(key, { ...result, at: Date.now() });
+    return result.valid ? null : blocked(tool, result.reason ?? "invalid");
+}

package/dist/server.js

@@ -1,6 +1,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { format, latestIndex, listCollections, searchCaptures } from "./api.js";
+import { format, latestIndex, listCollections, searchCaptures, captureHistory, indexStats, collectionDetails } from "./api.js";
+import { premiumRequired } from "./license.js";
 const text = (value) => ({ content: [{ type: "text", text: value }] });
 const textError = (t) => ({ content: [{ type: "text", text: t }], isError: true });
 const READ_ONLY = { readOnlyHint: true, openWorldHint: true };
@@ -51,5 +52,54 @@ export function createServer() {
             return errorText(error);
         }
     });
+    server.registerTool("capture_history", {
+        title: "Capture history",
+        description: "Trace every Common Crawl capture of a URL or domain in a given index with capture timestamps, status codes, MIME types, payload digests and lengths, restricted to an optional year range.",
+        inputSchema: z.object({ url: z.string().describe("URL or domain, e.g. example.com."), index: z.string().describe("Collection id, defaults to the latest.").optional(), match_type: z.string().describe("exact, prefix, host or domain.").optional(), from_year: z.number().describe("Only captures from this year onwards.").optional(), to_year: z.number().describe("Only captures up to this year.").optional(), limit: z.number().describe("Max records.").optional() }),
+        annotations: READ_ONLY,
+    }, async (args) => {
+        // ---- paywall: capture_history ----
+        const paywallMessage = await premiumRequired("capture_history");
+        if (paywallMessage)
+            return { content: [{ type: "text", text: paywallMessage }], isError: true };
+        // ---- paywall: end ----
+        try {
+            return text(await captureHistory(args));
+        }
+        catch (e) {
+            return textError(e instanceof Error ? `Error: ${e.message}` : String(e));
+        }
+    });
+    server.registerTool("index_stats", {
+        title: "Index stats",
+        description: "Profile a site as Common Crawl sees it: status code and MIME type distributions, 2xx share, scheme mix, URL path depth histogram and mean payload size over a sample of index records.",
+        inputSchema: z.object({ url: z.string().describe("URL or domain to profile."), index: z.string().describe("Collection id, defaults to the latest.").optional(), match_type: z.string().describe("exact, prefix, host or domain.").optional(), sample: z.number().describe("Max records to sample.").optional() }),
+        annotations: READ_ONLY,
+    }, async (args) => {
+        // ---- paywall: index_stats ----
+        const paywallMessage = await premiumRequired("index_stats");
+        if (paywallMessage)
+            return { content: [{ type: "text", text: paywallMessage }], isError: true };
+        // ---- paywall: end ----
+        try {
+            return text(await indexStats(args));
+        }
+        catch (e) {
+            return textError(e instanceof Error ? `Error: ${e.message}` : String(e));
+        }
+    });
+    server.registerTool("collection_details", {
+        title: "Collection details",
+        description: "Look up a specific Common Crawl collection by id with its timegate, CDX API endpoint, crawl window and total index size.",
+        inputSchema: z.object({ id: z.string().describe("Collection id, e.g. CC-MAIN-2026-39.") }),
+        annotations: READ_ONLY,
+    }, async (args) => {
+        try {
+            return text(await collectionDetails(args));
+        }
+        catch (e) {
+            return textError(e instanceof Error ? `Error: ${e.message}` : String(e));
+        }
+    });
     return server;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "commoncrawl-mcp",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Use this MCP server to Common Crawl index discovery for historical web captures. Tools include list indexes, latest index, search captures",
   "type": "module",
   "mcpName": "io.github.mrfentmen/commoncrawl-mcp",