@spicrawl/mcp 0.0.0-stage → 0.1.1

package/dist/client.js

@@ -0,0 +1,187 @@
+/**
+ * Thin client over the Spicrawl REST API.
+ *
+ * Every MCP tool ultimately calls one of these methods. The client owns three
+ * things the tools should not each re-implement: authentication (the caller's
+ * API key as a bearer token), the base URL, and turning the platform's RFC 7807
+ * problem envelope into an actionable Error the agent can act on rather than a
+ * bare HTTP status.
+ */
+import { createRequire } from "node:module";
+/**
+ * This package's version, read from package.json so the User-Agent, the MCP
+ * server's advertised version and the published package cannot drift. The
+ * path is relative to the compiled file (dist/client.js), and package.json
+ * ships with every npm package regardless of `files`.
+ */
+export const PACKAGE_VERSION = createRequire(import.meta.url)("../package.json").version;
+/**
+ * Sent on every API call. The API classifies the client surface from it
+ * (cli / mcp / sdk / other) and records it on each request-log row, so MCP
+ * traffic is countable. The product token must stay `spicrawl-mcp/`.
+ */
+export const USER_AGENT = `spicrawl-mcp/${PACKAGE_VERSION}`;
+/** The public Spicrawl API; set SPICRAWL_BASE_URL for a self-hosted or local stack. */
+const DEFAULT_BASE_URL = "https://api.spicrawl.com";
+const DEFAULT_TIMEOUT_MS = 120_000;
+/**
+ * Where the public docs live when nothing in the environment says otherwise:
+ * the root of their own host. llms.txt and the API's error doc_url links point
+ * here, and spicrawl_docs_read only reads URLs on the docs origin.
+ */
+const DEFAULT_DOCS_URL = "https://docs.spicrawl.com";
+const trimURL = (s) => (s ?? "").trim().replace(/\/+$/, "");
+/** An origin that serves the docs under /docs (a self-hosted API, the legacy SPICRAWL_DOCS_HOST). */
+const underDocs = (origin) => (origin.endsWith("/docs") ? origin : `${origin}/docs`);
+/**
+ * The docs base URL from the environment, no trailing slash; page paths are
+ * appended to it directly (`<base>/errors`, `<base>/llms.txt`):
+ *
+ *   1. SPICRAWL_DOCS_URL, the full base (`https://docs.spicrawl.com`, or
+ *      `http://host:8080/docs` on a self-hosted deployment);
+ *   2. SPICRAWL_DOCS_HOST + `/docs`, the legacy origin-only setting, which keeps
+ *      its old meaning so an existing env file does not silently change;
+ *   3. a self-hosted API (SPICRAWL_PUBLIC_BASE_URL, else SPICRAWL_BASE_URL) +
+ *      `/docs`, where every deployment serves its own docs;
+ *   4. https://docs.spicrawl.com, also when that API is the public one.
+ *
+ * Read from the raw environment, not the client's baseURL, so a stdio server
+ * with no base URL set points at the public docs rather than localhost.
+ */
+export function docsBaseFromEnv(env = process.env) {
+    const url = trimURL(env.SPICRAWL_DOCS_URL);
+    if (url)
+        return url;
+    const host = trimURL(env.SPICRAWL_DOCS_HOST);
+    if (host)
+        return underDocs(host);
+    const api = trimURL(env.SPICRAWL_PUBLIC_BASE_URL || env.SPICRAWL_BASE_URL);
+    if (api && api.toLowerCase() !== DEFAULT_BASE_URL)
+        return underDocs(api);
+    return DEFAULT_DOCS_URL;
+}
+/** A failed request, carrying the platform's own error code and guidance. */
+export class ILayerError extends Error {
+    code;
+    status;
+    retryable;
+    docUrl;
+    constructor(status, problem, fallback) {
+        const detail = problem.detail || problem.title || fallback;
+        super(detail);
+        this.name = "ILayerError";
+        this.code = problem.code || `HTTP_${status}`;
+        this.status = problem.status ?? status;
+        this.retryable = problem.retryable ?? status >= 500;
+        this.docUrl = problem.doc_url;
+    }
+}
+export class ILayerClient {
+    baseURL;
+    /** Base URL for URLs returned to the caller; never used for upstream calls. */
+    publicBaseURL;
+    /** The caller's key. Read by tools that hand out URLs carrying it (browser). */
+    apiKey;
+    /** Public docs base, e.g. https://docs.spicrawl.com; no trailing slash. */
+    docsBaseURL;
+    timeoutMs;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error("SPICRAWL_API_KEY is required. Set it to an Spicrawl API key (spicrawl_live_...) in the MCP server's environment.");
+        }
+        this.baseURL = (config.baseURL || DEFAULT_BASE_URL).replace(/\/+$/, "");
+        this.publicBaseURL = (config.publicBaseURL || this.baseURL).replace(/\/+$/, "");
+        this.apiKey = config.apiKey;
+        this.docsBaseURL = (config.docsBaseURL || docsBaseFromEnv()).replace(/\/+$/, "");
+        this.timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    }
+    /** POST /v1/scrape — retrieve, render, and extract one URL. */
+    async scrape(params) {
+        return this.request("POST", "/v1/scrape", params);
+    }
+    /** Any REST call. Tools for newer endpoints use this directly. */
+    async request(method, path, body) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+        let response;
+        try {
+            response = await fetch(`${this.baseURL}${path}`, {
+                method,
+                headers: {
+                    Authorization: `Bearer ${this.apiKey}`,
+                    Accept: "application/json",
+                    "User-Agent": USER_AGENT,
+                    ...(body ? { "Content-Type": "application/json" } : {}),
+                },
+                body: body ? JSON.stringify(body) : undefined,
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            if (err instanceof Error && err.name === "AbortError") {
+                throw new Error(`Request to ${path} timed out after ${this.timeoutMs} ms. A render or batch may need a longer timeout, or the API at ${this.baseURL} is unreachable.`);
+            }
+            throw new Error(`Could not reach the Spicrawl API at ${this.baseURL}${path}: ${err instanceof Error ? err.message : String(err)}. Check SPICRAWL_BASE_URL and that the server is running.`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        // A PDF (`response_format=pdf`) is bytes: read as text it would be corrupted.
+        if (response.ok && mediaType(response) === "application/pdf")
+            return pdfPayload(response);
+        const text = await response.text();
+        let parsed;
+        try {
+            parsed = text ? JSON.parse(text) : {};
+        }
+        catch {
+            // Not JSON: a scrape body (markdown/text/html) — or, on an error, a
+            // gateway's page, which the check below still reports as a failure.
+            parsed = response.ok || response.headers.get("x-engine") !== null ? { content: text } : { detail: text };
+        }
+        // A non-2xx status is only an error when the body is a platform problem.
+        // With `original_status` the API relays the target's own status (a 404
+        // page the caller allowed via `allowed_status_codes`) on a SUCCESSFUL
+        // scrape, and that content must reach the agent, not be reported as a
+        // failure.
+        // A completed scrape always carries X-Engine; a gateway's error page does not.
+        const relayedTargetStatus = response.headers.get("x-engine") !== null;
+        const isProblem = typeof parsed === "object" && parsed !== null && typeof parsed.code === "string";
+        if (!response.ok && (isProblem || !relayedTargetStatus)) {
+            throw new ILayerError(response.status, parsed, `Request failed with HTTP ${response.status}`);
+        }
+        return parsed;
+    }
+}
+/** The response's media type, lowercase, without parameters. */
+function mediaType(response) {
+    return (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase();
+}
+/**
+ * A `response_format=pdf` scrape answers with the file itself, and the envelope's
+ * fields as headers. The bytes come back base64 under `pdf`, with those fields
+ * beside it, for the scrape tool to hand over as a resource block (tools/pdf.ts).
+ */
+async function pdfPayload(response) {
+    const bytes = Buffer.from(await response.arrayBuffer());
+    const header = (name) => response.headers.get(name) ?? undefined;
+    const number = (name) => {
+        const v = header(name);
+        return v !== undefined && v !== "" && Number.isFinite(Number(v)) ? Number(v) : undefined;
+    };
+    const out = {
+        pdf: { content_type: "application/pdf", size_bytes: bytes.length, encoding: "base64", data: bytes.toString("base64") },
+    };
+    const fields = {
+        engine: header("x-engine"),
+        status: number("x-target-status"),
+        credits: number("x-credits-charged"),
+        final_url: header("x-final-url"),
+        request_id: header("x-request-id"),
+    };
+    for (const [k, v] of Object.entries(fields))
+        if (v !== undefined)
+            out[k] = v;
+    return out;
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAC1B,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CACjD,CAAC,OAAO,CAAC;AAEV;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,eAAe,EAAE,CAAC;AAE5D,uFAAuF;AACvF,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AACpD,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAErD,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAChF,qGAAqG;AACrG,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC;AAE7F;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,eAAe,CAAC,MAAyB,OAAO,CAAC,GAAG;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC3C,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IACpB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC7C,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC3E,IAAI,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,gBAAgB;QAAE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAaD,6EAA6E;AAC7E,MAAM,OAAO,WAAY,SAAQ,KAAK;IAC3B,IAAI,CAAS;IACb,MAAM,CAAS;IACf,SAAS,CAAU;IACnB,MAAM,CAAU;IAEzB,YAAY,MAAc,EAAE,OAAgB,EAAE,QAAgB;QAC5D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC;QAC3D,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,MAAM,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,IAAI,GAAG,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,CAAC;CACF;AAoBD,MAAM,OAAO,YAAY;IACd,OAAO,CAAS;IACzB,+EAA+E;IACtE,aAAa,CAAS;IAC/B,gFAAgF;IACvE,MAAM,CAAS;IACxB,2EAA2E;IAClE,WAAW,CAAS;IACZ,SAAS,CAAS;IAEnC,YAAY,MAAoB;QAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,kHAAkH,CACnH,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACjF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,MAAM,CAAC,MAA+B;QAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,OAAO,CACX,MAAc,EACd,IAAY,EACZ,IAA8B;QAE9B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,EAAE;gBAC/C,MAAM;gBACN,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;oBACtC,MAAM,EAAE,kBAAkB;oBAC1B,YAAY,EAAE,UAAU;oBACxB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxD;gBACD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7C,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CACb,cAAc,IAAI,oBAAoB,IAAI,CAAC,SAAS,mEAAmE,IAAI,CAAC,OAAO,kBAAkB,CACtJ,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,KAAK,CACb,uCAAuC,IAAI,CAAC,OAAO,GAAG,IAAI,KACxD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,2DAA2D,CAC5D,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,8EAA8E;QAC9E,IAAI,QAAQ,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,iBAAiB;YAAE,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE1F,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,oEAAoE;YACpE,oEAAoE;YACpE,MAAM,GAAG,QAAQ,CAAC,EAAE,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3G,CAAC;QAED,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,sEAAsE;QACtE,WAAW;QACX,+EAA+E;QAC/E,MAAM,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC;QACtE,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,OAAQ,MAAkB,CAAC,IAAI,KAAK,QAAQ,CAAC;QAChG,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,WAAW,CACnB,QAAQ,CAAC,MAAM,EACf,MAAiB,EACjB,4BAA4B,QAAQ,CAAC,MAAM,EAAE,CAC9C,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,gEAAgE;AAChE,SAAS,SAAS,CAAC,QAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACzF,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,UAAU,CAAC,QAAkB;IAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,CAAC,CAAC;IACF,MAAM,GAAG,GAA4B;QACnC,GAAG,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;KACvH,CAAC;IACF,MAAM,MAAM,GAA4B;QACtC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC;QAC1B,MAAM,EAAE,MAAM,CAAC,iBAAiB,CAAC;QACjC,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC;QACpC,SAAS,EAAE,MAAM,CAAC,aAAa,CAAC;QAChC,UAAU,EAAE,MAAM,CAAC,cAAc,CAAC;KACnC,CAAC;IACF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,IAAI,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/http.d.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+/**
+ * spicrawl-mcp-http — the Spicrawl MCP server over Streamable HTTP, so a user can
+ * connect an MCP client to a hosted URL instead of installing anything.
+ *
+ *   POST/GET/DELETE /mcp   MCP Streamable HTTP endpoint (Bearer auth)
+ *   GET /healthz           liveness, no auth
+ *
+ * Auth is the caller's own Spicrawl API key (`Authorization: Bearer spicrawl_live_...`);
+ * every tool call runs upstream under that key. One MCP server per session, and
+ * a session is pinned to the key that created it: a request presenting another
+ * key with that session id is refused (403), so no caller can run under someone
+ * else's key. An initialize is refused (401) unless the API accepts the key,
+ * and answered 503 when the API cannot be asked.
+ *
+ *   SPICRAWL_MCP_ADDR   listen address, default 127.0.0.1:8090
+ *   SPICRAWL_BASE_URL   upstream Spicrawl API, default http://127.0.0.1:8080
+ *   SPICRAWL_PUBLIC_BASE_URL  the API as remote callers reach it, used for URLs
+ *                     handed back (browser connect URL); default SPICRAWL_BASE_URL
+ *   SPICRAWL_DOCS_URL   the public docs base URL, read by the docs tools (never
+ *                     sent a key), e.g. http://HOST:8080/docs self-hosted; default
+ *                     SPICRAWL_DOCS_HOST + /docs (legacy), then a self-hosted
+ *                     SPICRAWL_PUBLIC_BASE_URL / SPICRAWL_BASE_URL + /docs, then
+ *                     https://docs.spicrawl.com (see docsBaseFromEnv)
+ *   SPICRAWL_MCP_SESSION_IDLE_MS  idle session eviction, default 1800000 (30 min)
+ */
+export {};

package/dist/http.js

@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+/**
+ * spicrawl-mcp-http — the Spicrawl MCP server over Streamable HTTP, so a user can
+ * connect an MCP client to a hosted URL instead of installing anything.
+ *
+ *   POST/GET/DELETE /mcp   MCP Streamable HTTP endpoint (Bearer auth)
+ *   GET /healthz           liveness, no auth
+ *
+ * Auth is the caller's own Spicrawl API key (`Authorization: Bearer spicrawl_live_...`);
+ * every tool call runs upstream under that key. One MCP server per session, and
+ * a session is pinned to the key that created it: a request presenting another
+ * key with that session id is refused (403), so no caller can run under someone
+ * else's key. An initialize is refused (401) unless the API accepts the key,
+ * and answered 503 when the API cannot be asked.
+ *
+ *   SPICRAWL_MCP_ADDR   listen address, default 127.0.0.1:8090
+ *   SPICRAWL_BASE_URL   upstream Spicrawl API, default http://127.0.0.1:8080
+ *   SPICRAWL_PUBLIC_BASE_URL  the API as remote callers reach it, used for URLs
+ *                     handed back (browser connect URL); default SPICRAWL_BASE_URL
+ *   SPICRAWL_DOCS_URL   the public docs base URL, read by the docs tools (never
+ *                     sent a key), e.g. http://HOST:8080/docs self-hosted; default
+ *                     SPICRAWL_DOCS_HOST + /docs (legacy), then a self-hosted
+ *                     SPICRAWL_PUBLIC_BASE_URL / SPICRAWL_BASE_URL + /docs, then
+ *                     https://docs.spicrawl.com (see docsBaseFromEnv)
+ *   SPICRAWL_MCP_SESSION_IDLE_MS  idle session eviction, default 1800000 (30 min)
+ */
+import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
+import { createServer } from "node:http";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
+import { ILayerClient, USER_AGENT } from "./client.js";
+import { buildServer } from "./server.js";
+const ADDR = process.env.SPICRAWL_MCP_ADDR || "127.0.0.1:8090";
+const BASE_URL = process.env.SPICRAWL_BASE_URL || "http://127.0.0.1:8080";
+const PUBLIC_BASE_URL = process.env.SPICRAWL_PUBLIC_BASE_URL || BASE_URL;
+const IDLE_MS = Number(process.env.SPICRAWL_MCP_SESSION_IDLE_MS) || 30 * 60 * 1000;
+const MAX_BODY = 4 * 1024 * 1024;
+const KEY_CHECK_TIMEOUT_MS = 5_000;
+const KEY_RE = /^spicrawl_(live|test)_[A-Za-z0-9_-]{8,}$/;
+const sessions = new Map();
+const log = (...a) => console.error(new Date().toISOString(), ...a);
+const hashKey = (k) => createHash("sha256").update(k).digest();
+function rpcError(res, status, code, message, extra = {}) {
+    if (res.headersSent)
+        return;
+    res.writeHead(status, { "Content-Type": "application/json", ...extra });
+    res.end(JSON.stringify({ jsonrpc: "2.0", error: { code, message }, id: null }));
+}
+function bearer(req) {
+    const h = req.headers.authorization;
+    if (!h)
+        return null;
+    const m = /^Bearer\s+(\S+)\s*$/i.exec(h);
+    return m && KEY_RE.test(m[1]) ? m[1] : null;
+}
+async function readJSON(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const c of req) {
+        size += c.length;
+        if (size > MAX_BODY)
+            throw new Error("body too large");
+        chunks.push(c);
+    }
+    const raw = Buffer.concat(chunks).toString("utf8");
+    return raw ? JSON.parse(raw) : undefined;
+}
+// The key's shape is checked locally, but only the API knows whether it exists.
+// Without asking, a revoked key opens a session and fails only on its first tool
+// call, which clients surface as a broken tool rather than a bad credential.
+// GET /v1/requests needs a valid key and no scope, so any non-401 answer below
+// 500 (a 403 on scope included) proves the key authenticated.
+async function checkKey(key) {
+    let res;
+    try {
+        res = await fetch(`${BASE_URL}/v1/requests?limit=1`, {
+            headers: { Authorization: `Bearer ${key}`, Accept: "application/json", "User-Agent": USER_AGENT },
+            signal: AbortSignal.timeout(KEY_CHECK_TIMEOUT_MS),
+        });
+    }
+    catch (err) {
+        log("key check: API unreachable:", err instanceof Error ? err.message : String(err));
+        return { ok: false, status: 503, message: "Service Unavailable: could not reach the Spicrawl API to verify the key; retry shortly" };
+    }
+    const text = await res.text().catch(() => "");
+    if (res.status === 401) {
+        let code = "ERR::AUTH::INVALID_KEY";
+        let detail = "the API key is invalid, revoked or expired";
+        try {
+            const p = JSON.parse(text);
+            if (typeof p.code === "string" && p.code)
+                code = p.code;
+            if (typeof p.detail === "string" && p.detail)
+                detail = p.detail;
+            else if (typeof p.title === "string" && p.title)
+                detail = p.title;
+        }
+        catch {
+            // Not a platform problem body; keep the generic wording.
+        }
+        return { ok: false, status: 401, message: `Unauthorized: ${code}: ${detail}` };
+    }
+    // A 5xx says nothing about the key, so refusing it as bad would send the
+    // caller off to rotate a working credential.
+    if (res.status >= 500) {
+        log(`key check: API answered HTTP ${res.status}`);
+        return { ok: false, status: 503, message: `Service Unavailable: the Spicrawl API answered HTTP ${res.status} while verifying the key; retry shortly` };
+    }
+    return { ok: true };
+}
+async function closeSession(id, why) {
+    const s = sessions.get(id);
+    if (!s)
+        return;
+    sessions.delete(id);
+    log(`session ${id.slice(0, 8)} closed (${why}); ${sessions.size} open`);
+    await s.transport.close().catch(() => { });
+    await s.server.close().catch(() => { });
+}
+async function handleMcp(req, res) {
+    const key = bearer(req);
+    if (!key) {
+        return rpcError(res, 401, -32001, "Unauthorized: send your Spicrawl API key as 'Authorization: Bearer spicrawl_live_...'", { "WWW-Authenticate": 'Bearer realm="spicrawl-mcp"' });
+    }
+    const keyHash = hashKey(key);
+    let body;
+    if (req.method === "POST") {
+        try {
+            body = await readJSON(req);
+        }
+        catch {
+            return rpcError(res, 400, -32700, "Parse error: body must be JSON (max 4 MiB)");
+        }
+    }
+    const sid = req.headers["mcp-session-id"];
+    if (typeof sid === "string" && sid) {
+        const s = sessions.get(sid);
+        if (!s)
+            return rpcError(res, 404, -32001, "Session not found or expired: re-initialize");
+        if (!timingSafeEqual(s.keyHash, keyHash)) {
+            log(`session ${sid.slice(0, 8)}: request with a different API key refused`);
+            return rpcError(res, 403, -32003, "Forbidden: this session belongs to a different API key");
+        }
+        s.lastSeen = Date.now();
+        await s.transport.handleRequest(req, res, body);
+        return;
+    }
+    if (req.method !== "POST" || !isInitializeRequest(body)) {
+        return rpcError(res, 400, -32000, "Bad Request: no Mcp-Session-Id; the first request must be initialize");
+    }
+    // Only a new session is checked: an existing one is already bound to this
+    // key's hash, and re-asking the API on every call would double its load.
+    const check = await checkKey(key);
+    if (!check.ok) {
+        return check.status === 401
+            ? rpcError(res, 401, -32001, check.message, { "WWW-Authenticate": 'Bearer realm="spicrawl-mcp"' })
+            : rpcError(res, 503, -32000, check.message, { "Retry-After": "5" });
+    }
+    const server = buildServer(new ILayerClient({ apiKey: key, baseURL: BASE_URL, publicBaseURL: PUBLIC_BASE_URL }));
+    const transport = new StreamableHTTPServerTransport({
+        sessionIdGenerator: () => randomUUID(),
+        onsessioninitialized: (id) => {
+            sessions.set(id, { transport, server, keyHash, lastSeen: Date.now() });
+            log(`session ${id.slice(0, 8)} opened; ${sessions.size} open`);
+        },
+    });
+    transport.onclose = () => {
+        if (transport.sessionId)
+            void closeSession(transport.sessionId, "closed");
+    };
+    await server.connect(transport);
+    await transport.handleRequest(req, res, body);
+}
+const http = createServer((req, res) => {
+    const path = (req.url || "/").split("?")[0];
+    if (path === "/healthz" && (req.method === "GET" || req.method === "HEAD")) {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(req.method === "HEAD" ? undefined : JSON.stringify({ status: "ok", sessions: sessions.size }));
+        return;
+    }
+    if (path !== "/mcp")
+        return rpcError(res, 404, -32601, "Not found: the MCP endpoint is /mcp");
+    if (!["GET", "POST", "DELETE"].includes(req.method || "")) {
+        return rpcError(res, 405, -32000, "Method not allowed", { Allow: "GET, POST, DELETE" });
+    }
+    handleMcp(req, res).catch((err) => {
+        log("request failed:", err instanceof Error ? err.message : String(err));
+        rpcError(res, 500, -32603, "Internal error");
+    });
+});
+// Idle eviction keeps memory bounded: a client that vanishes without DELETE
+// leaves its session behind otherwise.
+setInterval(() => {
+    const cutoff = Date.now() - IDLE_MS;
+    for (const [id, s] of sessions)
+        if (s.lastSeen < cutoff)
+            void closeSession(id, "idle");
+}, 60_000).unref();
+const i = ADDR.lastIndexOf(":");
+const host = i > 0 ? ADDR.slice(0, i).replace(/^\[|\]$/g, "") : "127.0.0.1";
+const port = Number(i >= 0 ? ADDR.slice(i + 1) : ADDR);
+if (!Number.isInteger(port) || port <= 0 || port > 65535) {
+    log(`invalid SPICRAWL_MCP_ADDR '${ADDR}' (want host:port)`);
+    process.exit(78);
+}
+http.listen(port, host, () => log(`spicrawl-mcp-http listening on http://${host}:${port}/mcp -> ${BASE_URL}`));
+http.on("error", (err) => {
+    log("spicrawl-mcp-http failed:", err.message);
+    process.exit(1);
+});
+const shutdown = () => {
+    log("shutting down");
+    http.close();
+    for (const id of [...sessions.keys()])
+        void closeSession(id, "shutdown");
+    setTimeout(() => process.exit(0), 2000).unref();
+};
+process.on("SIGTERM", shutdown);
+process.on("SIGINT", shutdown);
+//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,YAAY,EAA6C,MAAM,WAAW,CAAC;AACpF,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,CAAC;AAC/D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,uBAAuB,CAAC;AAC1E,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,QAAQ,CAAC;AACzE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACnF,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACjC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,MAAM,MAAM,GAAG,0CAA0C,CAAC;AAS1D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;AAE5C,MAAM,GAAG,GAAG,CAAC,GAAG,CAAY,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;AAC/E,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAEvE,SAAS,QAAQ,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAY,EAAE,OAAe,EAAE,QAAgC,EAAE;IACtH,IAAI,GAAG,CAAC,WAAW;QAAE,OAAO;IAC5B,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IACxE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,MAAM,CAAC,GAAoB;IAClC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC;IACpC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,GAAoB;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,IAAK,CAAY,CAAC,MAAM,CAAC;QAC7B,IAAI,IAAI,GAAG,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3C,CAAC;AAID,gFAAgF;AAChF,iFAAiF;AACjF,6EAA6E;AAC7E,+EAA+E;AAC/E,8DAA8D;AAC9D,KAAK,UAAU,QAAQ,CAAC,GAAW;IACjC,IAAI,GAAa,CAAC;IAClB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,sBAAsB,EAAE;YACnD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,GAAG,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,UAAU,EAAE;YACjG,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,oBAAoB,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,6BAA6B,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACrF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,wFAAwF,EAAE,CAAC;IACvI,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9C,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACvB,IAAI,IAAI,GAAG,wBAAwB,CAAC;QACpC,IAAI,MAAM,GAAG,4CAA4C,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA0D,CAAC;YACpF,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;YACxD,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM;gBAAE,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;iBAC3D,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK;gBAAE,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,iBAAiB,IAAI,KAAK,MAAM,EAAE,EAAE,CAAC;IACjF,CAAC;IACD,yEAAyE;IACzE,6CAA6C;IAC7C,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACtB,GAAG,CAAC,gCAAgC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,uDAAuD,GAAG,CAAC,MAAM,yCAAyC,EAAE,CAAC;IACzJ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,EAAU,EAAE,GAAW;IACjD,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3B,IAAI,CAAC,CAAC;QAAE,OAAO;IACf,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpB,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,OAAO,CAAC,CAAC;IACxE,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,GAAoB,EAAE,GAAmB;IAChE,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACxB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAC9B,uFAAuF,EACvF,EAAE,kBAAkB,EAAE,6BAA6B,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE7B,IAAI,IAAa,CAAC;IAClB,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,4CAA4C,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC;YAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,6CAA6C,CAAC,CAAC;QACzF,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,4CAA4C,CAAC,CAAC;YAC5E,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,wDAAwD,CAAC,CAAC;QAC9F,CAAC;QACD,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,MAAM,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAChD,OAAO;IACT,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,sEAAsE,CAAC,CAAC;IAC5G,CAAC;IAED,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACd,OAAO,KAAK,CAAC,MAAM,KAAK,GAAG;YACzB,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,kBAAkB,EAAE,6BAA6B,EAAE,CAAC;YAClG,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;IACjH,MAAM,SAAS,GAAkC,IAAI,6BAA6B,CAAC;QACjF,kBAAkB,EAAE,GAAG,EAAE,CAAC,UAAU,EAAE;QACtC,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE;YAC3B,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACvE,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,QAAQ,CAAC,IAAI,OAAO,CAAC,CAAC;QACjE,CAAC;KACF,CAAC,CAAC;IACH,SAAS,CAAC,OAAO,GAAG,GAAG,EAAE;QACvB,IAAI,SAAS,CAAC,SAAS;YAAE,KAAK,YAAY,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC5E,CAAC,CAAC;IACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;IACrC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;QAC3E,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC3D,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACvG,OAAO;IACT,CAAC;IACD,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC;IAC9F,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAC1D,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,oBAAoB,EAAE,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QAChC,GAAG,CAAC,iBAAiB,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACzE,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,uCAAuC;AACvC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACpC,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,QAAQ;QAAE,IAAI,CAAC,CAAC,QAAQ,GAAG,MAAM;YAAE,KAAK,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACzF,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;AAEnB,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;AAChC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;AAC5E,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACvD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;IACzD,GAAG,CAAC,8BAA8B,IAAI,oBAAoB,CAAC,CAAC;IAC5D,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACnB,CAAC;AAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,yCAAyC,IAAI,IAAI,IAAI,WAAW,QAAQ,EAAE,CAAC,CAAC,CAAC;AAC/G,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;IACvB,GAAG,CAAC,2BAA2B,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,MAAM,QAAQ,GAAG,GAAG,EAAE;IACpB,GAAG,CAAC,eAAe,CAAC,CAAC;IACrB,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,YAAY,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACzE,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;AAClD,CAAC,CAAC;AACF,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AAChC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+/**
+ * spicrawl-mcp — the Spicrawl web-data platform as native agent tools.
+ *
+ * A thin MCP wrapper over the Spicrawl REST API. Each tool maps to one endpoint;
+ * the caller's API key and the API base URL come from the environment, so the
+ * same server works against a local dev stack or a production deployment.
+ *
+ *   spicrawl-mcp           stdio server (the default; what `npx -y @spicrawl/mcp` runs)
+ *   spicrawl-mcp --http    Streamable HTTP server (dist/http.js; see there for its env)
+ *
+ * Every bin in package.json points at this file. `npx -y @spicrawl/mcp` can only
+ * pick a bin on its own when the package has a bin named `mcp` (too generic to
+ * install globally) or when every bin is the same file, so the HTTP server is a
+ * flag here rather than a second bin. `node dist/http.js` still runs it directly.
+ *
+ *   SPICRAWL_API_KEY   required — a Spicrawl key (spicrawl_live_...)
+ *   SPICRAWL_BASE_URL  optional — default https://api.spicrawl.com
+ *   SPICRAWL_PUBLIC_BASE_URL  optional — base for URLs handed back to the caller
+ *                    (browser connect URL); default SPICRAWL_BASE_URL
+ *   SPICRAWL_DOCS_URL optional — the public docs base URL (docs tools), e.g.
+ *                    http://HOST:8080/docs self-hosted; default SPICRAWL_DOCS_HOST
+ *                    + /docs (legacy), then a self-hosted SPICRAWL_PUBLIC_BASE_URL /
+ *                    SPICRAWL_BASE_URL + /docs, then https://docs.spicrawl.com
+ *
+ * Transport is stdio: this runs as a subprocess of an MCP client (Claude Code,
+ * Claude Desktop, Cursor, …), which is the local-integration case the MCP spec
+ * recommends stdio for.
+ */
+export {};

package/dist/index.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/**
+ * spicrawl-mcp — the Spicrawl web-data platform as native agent tools.
+ *
+ * A thin MCP wrapper over the Spicrawl REST API. Each tool maps to one endpoint;
+ * the caller's API key and the API base URL come from the environment, so the
+ * same server works against a local dev stack or a production deployment.
+ *
+ *   spicrawl-mcp           stdio server (the default; what `npx -y @spicrawl/mcp` runs)
+ *   spicrawl-mcp --http    Streamable HTTP server (dist/http.js; see there for its env)
+ *
+ * Every bin in package.json points at this file. `npx -y @spicrawl/mcp` can only
+ * pick a bin on its own when the package has a bin named `mcp` (too generic to
+ * install globally) or when every bin is the same file, so the HTTP server is a
+ * flag here rather than a second bin. `node dist/http.js` still runs it directly.
+ *
+ *   SPICRAWL_API_KEY   required — a Spicrawl key (spicrawl_live_...)
+ *   SPICRAWL_BASE_URL  optional — default https://api.spicrawl.com
+ *   SPICRAWL_PUBLIC_BASE_URL  optional — base for URLs handed back to the caller
+ *                    (browser connect URL); default SPICRAWL_BASE_URL
+ *   SPICRAWL_DOCS_URL optional — the public docs base URL (docs tools), e.g.
+ *                    http://HOST:8080/docs self-hosted; default SPICRAWL_DOCS_HOST
+ *                    + /docs (legacy), then a self-hosted SPICRAWL_PUBLIC_BASE_URL /
+ *                    SPICRAWL_BASE_URL + /docs, then https://docs.spicrawl.com
+ *
+ * Transport is stdio: this runs as a subprocess of an MCP client (Claude Code,
+ * Claude Desktop, Cursor, …), which is the local-integration case the MCP spec
+ * recommends stdio for.
+ */
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { ILayerClient, PACKAGE_VERSION } from "./client.js";
+import { buildServer } from "./server.js";
+const USAGE = `spicrawl-mcp ${PACKAGE_VERSION} — the Spicrawl MCP server
+
+Usage:
+  spicrawl-mcp            run over stdio (for an MCP client; needs SPICRAWL_API_KEY)
+  spicrawl-mcp --http     run the Streamable HTTP server (SPICRAWL_MCP_ADDR, default 127.0.0.1:8090)
+  spicrawl-mcp --version  print the version
+  spicrawl-mcp --help     print this help
+
+Docs: https://docs.spicrawl.com/agents/mcp
+`;
+async function stdio() {
+    const server = buildServer(new ILayerClient({
+        apiKey: process.env.SPICRAWL_API_KEY ?? "",
+        baseURL: process.env.SPICRAWL_BASE_URL,
+        publicBaseURL: process.env.SPICRAWL_PUBLIC_BASE_URL,
+    }));
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    // stdout is the transport; logs go to stderr.
+    console.error(`spicrawl-mcp ${PACKAGE_VERSION} ready (stdio)`);
+}
+async function main(args) {
+    if (args.includes("--help") || args.includes("-h")) {
+        process.stdout.write(USAGE);
+        return;
+    }
+    if (args.includes("--version") || args.includes("-v")) {
+        process.stdout.write(`${PACKAGE_VERSION}\n`);
+        return;
+    }
+    const unknown = args.filter((a) => a !== "--http");
+    if (unknown.length > 0) {
+        console.error(`spicrawl-mcp: unknown argument ${unknown[0]}\n\n${USAGE}`);
+        process.exit(2);
+    }
+    if (args.includes("--http")) {
+        // dist/http.js starts listening when it is loaded.
+        await import("./http.js");
+        return;
+    }
+    await stdio();
+}
+main(process.argv.slice(2)).catch((err) => {
+    console.error("spicrawl-mcp failed to start:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,KAAK,GAAG,gBAAgB,eAAe;;;;;;;;;CAS5C,CAAC;AAEF,KAAK,UAAU,KAAK;IAClB,MAAM,MAAM,GAAG,WAAW,CACxB,IAAI,YAAY,CAAC;QACf,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE;QAC1C,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QACtC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;KACpD,CAAC,CACH,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,8CAA8C;IAC9C,OAAO,CAAC,KAAK,CAAC,gBAAgB,eAAe,gBAAgB,CAAC,CAAC;AACjE,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAc;IAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,eAAe,IAAI,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,kCAAkC,OAAO,CAAC,CAAC,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,mDAAmD;QACnD,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1B,OAAO;IACT,CAAC;IACD,MAAM,KAAK,EAAE,CAAC;AAChB,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACxC,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/server.d.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { ILayerClient } from "./client.js";
+export declare const SERVER_NAME = "spicrawl-mcp-server";
+export declare const SERVER_VERSION: string;
+/**
+ * Wraps a raw zod shape into a strict object: unknown keys are a validation
+ * error naming them (with a did-you-mean), and the JSON Schema carries
+ * `additionalProperties: false`. Descriptions and defaults are untouched.
+ */
+export declare function strictArgs<T extends z.ZodRawShape>(shape: T): z.ZodObject<T, "strict", z.ZodTypeAny, z.objectUtil.addQuestionMarks<z.baseObjectOutputType<T>, any> extends infer T_1 ? { [k in keyof T_1]: T_1[k]; } : never, z.baseObjectInputType<T> extends infer T_2 ? { [k_1 in keyof T_2]: T_2[k_1]; } : never>;
+/**
+ * One MCP server bound to one caller's API key. The stdio entry builds one for
+ * the process; the HTTP entry builds one per session, from that session's key.
+ */
+export declare function buildServer(client: ILayerClient): McpServer;