@ooky/sdk 0.1.0 → 0.6.0

package/src/express.js

@@ -1,20 +1,58 @@
 /**
  * Express adapter — `app.use(ookyMiddleware({ apiKey, domain }))`.
  *
- * Intercepts the well-known AI paths and serves the manifest. For every
- * request (manifest or not), checks the User-Agent against the bot registry
- * and fires a fire-and-forget event when a bot is detected.
+ * Intercepts the well-known AI paths and serves the manifest, answers MCP
+ * tool invocations on POST /mcp, and for every request checks the
+ * User-Agent against the bot registry — firing a fire-and-forget event when
+ * a bot is detected, or an ai_referral event when a human arrives from an
+ * AI platform (ChatGPT, Perplexity, Claude, …).
  */
 
-import { createOokyHandler } from "./core.js";
+import {
+  createOokyHandler,
+  logMiddlewareDisabled,
+  MAX_MCP_BODY_BYTES,
+  MAX_UA_LENGTH,
+  MAX_PATH_LENGTH,
+  clampString,
+} from "./core.js";
 
 export function ookyMiddleware(options) {
-  const handler = createOokyHandler(options);
+  let handler;
+  try {
+    handler = createOokyHandler(options);
+  } catch (err) {
+    // Construction failed (almost always missing OOKY_API_KEY / OOKY_DOMAIN).
+    // This middleware is mounted on every request, so throwing here would take
+    // down the customer's whole app. Fail safe: log loudly, return a no-op that
+    // simply calls next().
+    logMiddlewareDisabled(err);
+    return function ookyHandlerDisabled(req, res, next) {
+      return next();
+    };
+  }
 
   return async function ookyHandler(req, res, next) {
-    const ua = req.headers["user-agent"] || "";
+    const rawUa = req.headers["user-agent"] || "";
     const path = req.path || req.url || "/";
-    const bot = handler.detectBot(ua);
+    // Defensive caps on untrusted strings before they enter the event payload.
+    const ua = clampString(rawUa, MAX_UA_LENGTH);
+    const pagePath = clampString(path.split("?")[0] || "/", MAX_PATH_LENGTH);
+    const method = req.method || "GET";
+    const country = countryFromExpress(req);
+
+    // detectBot / matchPath run on EVERY request. By contract they never
+    // throw (a malformed bot registry is sanitised in core), but a middleware
+    // must not be able to crash the customer's process under ANY circumstance
+    // — so we degrade to pass-through if anything here throws.
+    let bot = null;
+    let kind = null;
+    try {
+      bot = handler.detectBot(ua);
+      kind = handler.matchPath(path);
+    } catch {
+      return next();
+    }
 
     // Fire bot event regardless of whether we serve the manifest. The Ooky
     // dashboard tracks bot visits across all routes, not just /llms.txt.
@@ -24,24 +62,126 @@ export function ookyMiddleware(options) {
       handler.recordEvent({
         bot: { name: bot.name, verified: false, ua_string: ua },
         request: {
-          page_path: path.split("?")[0] || "/",
-          method: req.method || "GET",
+          page_path: pagePath,
+          method,
+          manifest_file: kind || null,
         },
+        geo: country ? { country } : null,
       });
+    } else {
+      // Human traffic: attribute visits referred by AI platforms.
+      const referral = handler.detectReferral(
+        req.headers["referer"] || req.headers["referrer"],
+        req.query?.utm_source ?? parseUtmSource(req.url)
+      );
+      if (referral) {
+        handler.recordEvent({
+          event_type: "ai_referral",
+          referral: {
+            source: referral.source,
+            referrer_url: referral.referrerUrl,
+            detection_method: referral.method,
+          },
+          request: { page_path: pagePath, method },
+          geo: country ? { country } : null,
+        });
+      }
     }
 
-    const kind = handler.matchPath(path);
     if (!kind) return next();
 
-    const { status, headers, body } = await handler.serveManifest(kind);
-    res.status(status);
-    for (const [k, v] of Object.entries(headers)) {
-      res.setHeader(k, v);
-    }
-    if (typeof body === "string") {
-      res.send(body);
-    } else {
-      res.json(body);
+    try {
+      let result;
+      if (kind === "mcp" && req.method === "POST") {
+        result = await handler.handleMcpInvocation(await readJsonBody(req));
+      } else if (kind === "mcp" && req.method === "OPTIONS") {
+        result = { status: 204, headers: corsPreflightHeaders(), body: "" };
+      } else {
+        result = await handler.serveManifest(kind);
+      }
+
+      const { status, headers, body } = result;
+      if (res.headersSent) return;
+      res.status(status);
+      for (const [k, v] of Object.entries(headers)) {
+        res.setHeader(k, v);
+      }
+      if (body === null || body === undefined) {
+        res.end(); // e.g. 202 for MCP notifications
+      } else if (typeof body === "string") {
+        res.send(body);
+      } else {
+        res.json(body);
+      }
+    } catch (err) {
+      // serveManifest never throws by contract, but a middleware must not be
+      // able to crash the customer's app under any circumstances.
+      if (!res.headersSent) next(err);
     }
   };
 }
+
+function corsPreflightHeaders() {
+  return {
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
+    "Access-Control-Allow-Headers": "Content-Type",
+  };
+}
+
+/**
+ * Best-effort country from edge/CDN headers so the dashboard's geo panel
+ * isn't empty for SDK customers. Cloudflare sets `cf-ipcountry`; Vercel sets
+ * `x-vercel-ip-country`. "XX"/"T1" are CF placeholders for unknown/Tor — drop
+ * them. Returns a 2-letter uppercase code or null.
+ */
+function countryFromExpress(req) {
+  const raw =
+    req.headers["cf-ipcountry"] ||
+    req.headers["x-vercel-ip-country"] ||
+    req.headers["x-appengine-country"] ||
+    "";
+  return normalizeCountry(raw);
+}
+
+function normalizeCountry(raw) {
+  if (!raw || typeof raw !== "string") return null;
+  const code = raw.trim().toUpperCase();
+  if (code.length !== 2 || code === "XX" || code === "T1") return null;
+  return code;
+}
+
+/** Fallback utm_source extraction when req.query isn't populated. */
+function parseUtmSource(url) {
+  if (!url || !url.includes("utm_source=")) return null;
+  try {
+    return new URL(url, "http://localhost").searchParams.get("utm_source");
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read the request body as JSON. Uses req.body when a body parser already
+ * consumed the stream (e.g. express.json() mounted before us); otherwise
+ * reads the raw stream with a size cap. Returns null on any parse failure —
+ * handleMcpInvocation turns that into a 400.
+ */
+async function readJsonBody(req) {
+  if (req.body && typeof req.body === "object" && Object.keys(req.body).length > 0) {
+    return req.body;
+  }
+  try {
+    let size = 0;
+    const chunks = [];
+    for await (const chunk of req) {
+      size += chunk.length;
+      if (size > MAX_MCP_BODY_BYTES) return null;
+      chunks.push(chunk);
+    }
+    if (chunks.length === 0) return null;
+    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+  } catch {
+    return null;
+  }
+}

package/src/index.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Type declarations for @ooky/sdk. Hand-written — the package ships plain
+ * ESM JavaScript with no build step; keep these in sync with src/core.js.
+ */
+
+export type ManifestKind = "llms" | "llms-full" | "manifest" | "agents" | "mcp";
+
+export interface BotEntry {
+  name: string;
+  /** Case-insensitive substring matched against the User-Agent. */
+  pattern: string;
+  category?: "ai" | "search" | "social" | "other" | string;
+  verified?: boolean;
+}
+
+export interface OokyErrorContext {
+  op: "recordEvent" | "serveManifest" | "refreshBotRegistry";
+  /** HTTP status when the failure was a non-2xx upstream response. */
+  status?: number;
+  kind?: ManifestKind;
+  /** True when events were dropped by the maxEventsPerMinute token bucket. */
+  throttled?: boolean;
+}
+
+export interface AIReferral {
+  source: string;
+  referrerUrl: string | null;
+  method: "referer_header" | "utm_param";
+}
+
+export interface OokyHandlerOptions {
+  /** Bearer token from the dashboard (`ooky_sk_*`). */
+  apiKey: string;
+  /** Domain registered in Ooky, e.g. "acme.com". */
+  domain: string;
+  /** Override the Ooky API base URL. Default: https://api.ooky.ai/api */
+  apiBase?: string;
+  /** Override the manifest CDN base URL. Default: `${apiBase}/public/manifest` */
+  cdnBase?: string;
+  /** Override the bot registry. Default: the built-in list. */
+  bots?: BotEntry[];
+  /** Periodically refresh the bot registry from /api/public/bots. Default: true */
+  autoRefreshBots?: boolean;
+  /** Upstream fetch timeout in ms (manifest + registry + events). Default: 10000 */
+  fetchTimeoutMs?: number;
+  /** In-memory manifest cache TTL in ms. 0 disables caching. Default: 300000 */
+  manifestCacheTtlMs?: number;
+  /**
+   * Called for every failure the SDK swallows (event POST rejections and
+   * non-2xx responses — e.g. 401 after a key rotation — manifest fetch
+   * failures, registry refresh failures). Default: silent.
+   */
+  onError?: (error: Error, context: OokyErrorContext) => void;
+  /**
+   * Token-bucket cap on event POSTs per minute (bot-storm insurance).
+   * Drops are reported via onError. Pass Infinity to disable. Default: 300
+   */
+  maxEventsPerMinute?: number;
+}
+
+export interface ManifestResponse {
+  status: number;
+  headers: Record<string, string>;
+  /** String for text kinds (llms, llms-full, agents); object for JSON kinds. */
+  body: string | Record<string, unknown>;
+}
+
+export interface OokyEventPayload {
+  event_id?: string;
+  timestamp?: string;
+  /** Set to "ai_referral" (with `referral`) for AI-platform referral visits. */
+  event_type?: string;
+  referral?: {
+    source: string;
+    referrer_url?: string | null;
+    detection_method?: string;
+  } | null;
+  bot?: { name: string; verified?: boolean; ua_string?: string } | null;
+  request?: {
+    page_path?: string;
+    method?: string;
+    manifest_version?: string | null;
+    manifest_file?: string | null;
+  } | null;
+  session?: Record<string, unknown> | null;
+  geo?: { country?: string | null } | null;
+  serve?: Record<string, unknown> | null;
+}
+
+export interface OokyHandler {
+  matchPath(path: string): ManifestKind | null;
+  detectBot(userAgent: string): BotEntry | null;
+  /** Detect a human visit referred from an AI platform (Referer / utm_source). */
+  detectReferral(referer?: string | null, utmSource?: string | null): AIReferral | null;
+  serveManifest(kind: ManifestKind): Promise<ManifestResponse>;
+  /**
+   * Answer an MCP request (POST /mcp or /.well-known/mcp). Speaks BOTH
+   * protocols:
+   *  - Standard MCP — JSON-RPC 2.0 over streamable HTTP (`initialize`,
+   *    `tools/list`, `tools/call`, `ping`). What real MCP clients (Claude,
+   *    MCP Inspector) use; detected by `jsonrpc: "2.0"` on the body. A `null`
+   *    body (unparseable JSON) returns a JSON-RPC parse error (-32700).
+   *  - Legacy Ooky — `{ tool, arguments }` → `{ result }`, kept for
+   *    Worker-tier compatibility.
+   */
+  handleMcpInvocation(body: unknown): Promise<ManifestResponse>;
+  /** Fire-and-forget; never rejects. Hand to `event.waitUntil()` on edge runtimes. */
+  recordEvent(payload: OokyEventPayload): Promise<unknown>;
+  refreshBotRegistry(force?: boolean): Promise<BotEntry[]>;
+  /** In-flight registry refresh, if any. Hand to `event.waitUntil()` on edge runtimes. */
+  pendingBotRefresh(): Promise<BotEntry[]> | null;
+}
+
+export const SDK_VERSION: string;
+/** Best-effort runtime tag stamped into the `X-Ooky-Sdk` header ("node" | "edge" | "web"). */
+export const SDK_RUNTIME: string;
+/** Max MCP request body the adapters accept (64KB). */
+export const MAX_MCP_BODY_BYTES: number;
+/** Defensive cap on the bot User-Agent copied into events (1024). */
+export const MAX_UA_LENGTH: number;
+/** Defensive cap on the request path copied into events (2048). */
+export const MAX_PATH_LENGTH: number;
+/** Truncate an untrusted string to `max` chars; non-strings pass through. */
+export function clampString<T>(value: T, max: number): T;
+export function createOokyHandler(options: OokyHandlerOptions): OokyHandler;

package/src/mcp.js

@@ -0,0 +1,127 @@
+/**
+ * Stateless MCP (Model Context Protocol) JSON-RPC 2.0 handler.
+ *
+ * Real MCP clients (Claude, MCP Inspector, ChatGPT connectors) speak
+ * JSON-RPC over streamable HTTP: initialize → notifications/initialized →
+ * tools/list → tools/call. This module implements the stateless subset —
+ * each POST is handled independently, single JSON response per request (the
+ * spec allows servers to answer with plain application/json instead of an
+ * SSE stream).
+ *
+ * Mirrors worker/src/mcp.js — when you change the protocol surface here,
+ * change it there too. Runtime-agnostic: no Node-only APIs.
+ */
+
+export const MCP_PROTOCOL_VERSION = "2025-03-26";
+
+// JSON-RPC 2.0 error codes.
+const PARSE_ERROR = -32700;
+const INVALID_REQUEST = -32600;
+const METHOD_NOT_FOUND = -32601;
+const INVALID_PARAMS = -32602;
+const INTERNAL_ERROR = -32603;
+
+/**
+ * Handle one MCP JSON-RPC message.
+ *
+ * @param {unknown} message  Parsed request body (or null on JSON parse failure).
+ * @param {object} server
+ * @param {string} server.name           serverInfo.name
+ * @param {string} server.version        serverInfo.version
+ * @param {Array}  server.tools          Tool descriptors ({ name, description, inputSchema }).
+ * @param {Function} server.callTool     async (name, args) → result object, or throws
+ *                                       McpToolError for invalid-params-class failures.
+ * @returns {{ status: number, body: object|null }}  body null → empty response (notifications).
+ */
+export async function handleMcpJsonRpc(message, server) {
+  if (message === null || message === undefined) {
+    return rpcError(null, PARSE_ERROR, "Parse error: body must be valid JSON");
+  }
+  // JSON-RPC batches were removed from MCP in the 2025-06-18 revision; we
+  // never supported them, so reject explicitly.
+  if (Array.isArray(message)) {
+    return rpcError(null, INVALID_REQUEST, "Batch requests are not supported");
+  }
+  if (typeof message !== "object" || message.jsonrpc !== "2.0" || typeof message.method !== "string") {
+    return rpcError(message?.id ?? null, INVALID_REQUEST, "Invalid JSON-RPC 2.0 request");
+  }
+
+  const { id, method, params } = message;
+  const isNotification = id === undefined || id === null;
+
+  // Notifications get 202 Accepted with no body (streamable HTTP transport).
+  if (method.startsWith("notifications/")) {
+    return { status: 202, body: null };
+  }
+  if (isNotification) {
+    // Requests we'd have to answer but can't address — accept and drop.
+    return { status: 202, body: null };
+  }
+
+  try {
+    switch (method) {
+      case "initialize":
+        return rpcResult(id, {
+          protocolVersion: negotiateVersion(params?.protocolVersion),
+          capabilities: { tools: {} },
+          serverInfo: { name: server.name, version: server.version },
+        });
+
+      case "ping":
+        return rpcResult(id, {});
+
+      case "tools/list":
+        return rpcResult(id, { tools: server.tools });
+
+      case "tools/call": {
+        const name = params?.name;
+        if (!name || typeof name !== "string") {
+          return rpcError(id, INVALID_PARAMS, "tools/call requires params.name");
+        }
+        if (!server.tools.some((t) => t.name === name)) {
+          return rpcError(id, INVALID_PARAMS, `Unknown tool: ${name}`);
+        }
+        const data = await server.callTool(name, params?.arguments || {});
+        return rpcResult(id, {
+          content: [{ type: "text", text: JSON.stringify(data) }],
+          isError: false,
+        });
+      }
+
+      default:
+        return rpcError(id, METHOD_NOT_FOUND, `Method not found: ${method}`);
+    }
+  } catch (err) {
+    if (err instanceof McpToolError) {
+      // Tool execution failures are reported in-band per the MCP spec so the
+      // LLM can see them, not as protocol errors.
+      return rpcResult(id, {
+        content: [{ type: "text", text: err.message }],
+        isError: true,
+      });
+    }
+    return rpcError(id, INTERNAL_ERROR, "Internal error");
+  }
+}
+
+/** Throw from callTool to report a tool-level failure in-band (isError: true). */
+export class McpToolError extends Error {}
+
+/**
+ * Echo the client's requested protocol version when we can speak it,
+ * otherwise offer ours (the client disconnects if that's unacceptable).
+ */
+function negotiateVersion(requested) {
+  if (typeof requested === "string" && requested <= MCP_PROTOCOL_VERSION) {
+    return requested;
+  }
+  return MCP_PROTOCOL_VERSION;
+}
+
+function rpcResult(id, result) {
+  return { status: 200, body: { jsonrpc: "2.0", id, result } };
+}
+
+function rpcError(id, code, message) {
+  return { status: 200, body: { jsonrpc: "2.0", id, error: { code, message } } };
+}

package/src/next.d.ts

@@ -0,0 +1,12 @@
+import type { OokyHandlerOptions } from "./index";
+
+/**
+ * Next.js middleware (Node or Edge runtime).
+ * Returns a Response for the well-known AI paths, undefined to fall through.
+ */
+export function ookyMiddleware(
+  options: OokyHandlerOptions
+): (
+  request: Request,
+  event?: { waitUntil(promise: Promise<unknown>): void }
+) => Promise<Response | undefined>;

package/src/next.js

@@ -8,36 +8,155 @@
  *
  * Returns a function with the Next.js middleware signature: it receives a
  * NextRequest-like object (compatible Web Request) and returns a Response,
- * or undefined to fall through to the next middleware/route.
+ * or undefined to fall through to the next middleware/route. Serves the
+ * well-known AI paths, answers MCP tool invocations on POST /mcp, and fires
+ * bot / ai_referral analytics events.
  */
 
-import { createOokyHandler } from "./core.js";
+import { createOokyHandler, logMiddlewareDisabled, MAX_MCP_BODY_BYTES, MAX_UA_LENGTH, MAX_PATH_LENGTH, clampString } from "./core.js";
 
 export function ookyMiddleware(options) {
-  const handler = createOokyHandler(options);
+  let handler;
+  try {
+    handler = createOokyHandler(options);
+  } catch (err) {
+    // Construction failed (almost always missing OOKY_API_KEY / OOKY_DOMAIN).
+    // The middleware runs on every request, so throwing here would 500 the
+    // customer's entire site. Fail safe: log loudly, return a pass-through.
+    logMiddlewareDisabled(err);
+    return function ookyNextMiddlewareDisabled() {
+      return undefined; // Next continues to the route — app unaffected.
+    };
+  }
 
-  return async function ookyNextMiddleware(request) {
+  return async function ookyNextMiddleware(request, event) {
     const url = new URL(request.url);
-    const ua = request.headers.get("user-agent") || "";
-    const bot = handler.detectBot(ua);
+    const rawUa = request.headers.get("user-agent") || "";
+    const ua = clampString(rawUa, MAX_UA_LENGTH);
+    const pagePath = clampString(url.pathname || "/", MAX_PATH_LENGTH);
+    const method = request.method || "GET";
+    const country = countryFromRequest(request);
+
+    const waitUntil =
+      event && typeof event.waitUntil === "function"
+        ? (p) => event.waitUntil(p)
+        : () => {};
+
+    // detectBot / matchPath run on EVERY request. They never throw by
+    // contract (the registry is sanitised in core), but a middleware must
+    // never be able to crash the customer's app — degrade to pass-through.
+    let bot = null;
+    let kind = null;
+    try {
+      bot = handler.detectBot(ua);
+      kind = handler.matchPath(url.pathname);
+    } catch {
+      return undefined;
+    }
+
+    // detectBot may have kicked off the hourly bot-registry refresh; keep it
+    // alive past the response on edge runtimes.
+    const refresh = handler.pendingBotRefresh();
+    if (refresh) waitUntil(refresh);
 
     // `verified: false` — UA-only matching can't prove bot identity. The
     // Worker tier does IP + reverse-DNS verification; the SDK can't.
     if (bot) {
-      handler.recordEvent({
-        bot: { name: bot.name, verified: false, ua_string: ua },
-        request: {
-          page_path: url.pathname || "/",
-          method: request.method || "GET",
-        },
-      });
+      waitUntil(
+        handler.recordEvent({
+          bot: { name: bot.name, verified: false, ua_string: ua },
+          request: {
+            page_path: pagePath,
+            method,
+            manifest_file: kind || null,
+          },
+          geo: country ? { country } : null,
+        })
+      );
+    } else {
+      // Human traffic: attribute visits referred by AI platforms.
+      const referral = handler.detectReferral(
+        request.headers.get("referer"),
+        url.searchParams.get("utm_source")
+      );
+      if (referral) {
+        waitUntil(
+          handler.recordEvent({
+            event_type: "ai_referral",
+            referral: {
+              source: referral.source,
+              referrer_url: referral.referrerUrl,
+              detection_method: referral.method,
+            },
+            request: { page_path: pagePath, method },
+            geo: country ? { country } : null,
+          })
+        );
+      }
     }
 
-    const kind = handler.matchPath(url.pathname);
     if (!kind) return undefined;
 
-    const { status, headers, body } = await handler.serveManifest(kind);
-    const responseBody = typeof body === "string" ? body : JSON.stringify(body);
+    let result;
+    if (kind === "mcp" && request.method === "POST") {
+      // Cap the MCP request body (parity with the Express adapter's 64KB
+      // streaming cap). Reject oversized bodies via Content-Length before
+      // buffering the whole thing into memory with request.json().
+      const declaredLength = Number(request.headers.get("content-length"));
+      if (Number.isFinite(declaredLength) && declaredLength > MAX_MCP_BODY_BYTES) {
+        return new Response(JSON.stringify({ error: "Request body too large" }), {
+          status: 413,
+          headers: { "Content-Type": "application/json; charset=utf-8" },
+        });
+      }
+      const body = await request.json().catch(() => null);
+      result = await handler.handleMcpInvocation(body);
+    } else if (kind === "mcp" && request.method === "OPTIONS") {
+      return new Response(null, {
+        status: 204,
+        headers: {
+          "Access-Control-Allow-Origin": "*",
+          "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
+          "Access-Control-Allow-Headers": "Content-Type",
+        },
+      });
+    } else {
+      result = await handler.serveManifest(kind);
+    }
+
+    const { status, headers, body } = result;
+    const responseBody =
+      body === null || body === undefined
+        ? null // e.g. 202 for MCP notifications
+        : typeof body === "string"
+          ? body
+          : JSON.stringify(body);
     return new Response(responseBody, { status, headers });
   };
 }
+
+/**
+ * Best-effort country from edge headers so the dashboard's geo panel isn't
+ * empty for SDK customers. Prefers `request.geo?.country` (Vercel populates it
+ * on NextRequest), then Cloudflare `cf-ipcountry` / Vercel
+ * `x-vercel-ip-country` headers. "XX"/"T1" are CF unknown/Tor placeholders.
+ */
+function countryFromRequest(request) {
+  const geoCountry = request && request.geo && request.geo.country;
+  if (geoCountry) {
+    const norm = normalizeCountry(geoCountry);
+    if (norm) return norm;
+  }
+  const raw =
+    request.headers.get("cf-ipcountry") ||
+    request.headers.get("x-vercel-ip-country") ||
+    "";
+  return normalizeCountry(raw);
+}
+
+function normalizeCountry(raw) {
+  if (!raw || typeof raw !== "string") return null;
+  const code = raw.trim().toUpperCase();
+  if (code.length !== 2 || code === "XX" || code === "T1") return null;
+  return code;
+}