@mentionable/tracker 0.1.0

package/README.md

@@ -0,0 +1,144 @@
+# @mentionable/tracker
+
+Track AI bot crawls on your website. Monitor when GPTBot, ClaudeBot, PerplexityBot, Grok and others crawl your pages.
+
+See which pages AI models are indexing, how often they come back, and correlate crawl data with your AI visibility on [Mentionable](https://mentionable.io).
+
+## Install
+
+```bash
+npm install @mentionable/tracker
+```
+
+## Quick Start
+
+### Next.js
+
+```ts
+// middleware.ts
+import { withMentionable } from '@mentionable/tracker/next'
+
+export default withMentionable({
+  apiKey: process.env.MENTIONABLE_API_KEY!,
+})
+
+export const config = {
+  matcher: ['/((?!_next/static|_next/image|favicon.ico).*)'],
+}
+```
+
+That's it. Every AI bot crawl is now tracked in your [Mentionable dashboard](https://mentionable.io).
+
+### Cloudflare Worker
+
+```ts
+// src/index.ts
+import { withMentionable } from '@mentionable/tracker/cloudflare'
+
+export default withMentionable({
+  apiKey: 'mtbl_xxxxx', // or use env: true to read from MENTIONABLE_API_KEY env var
+})
+```
+
+```bash
+npx wrangler deploy
+```
+
+Works in front of any origin: Shopify, WordPress, custom backends. The Worker proxies the request and reports crawl data without adding any latency.
+
+### Existing Middleware (Next.js)
+
+If you already have a `middleware.ts`, use the handler directly:
+
+```ts
+// middleware.ts
+import { NextResponse } from 'next/server'
+import { createMentionable } from '@mentionable/tracker/next'
+
+const mentionable = createMentionable({
+  apiKey: process.env.MENTIONABLE_API_KEY!,
+})
+
+export async function middleware(req) {
+  // your existing logic
+  mentionable.track(req)
+
+  return NextResponse.next()
+}
+```
+
+### Existing Cloudflare Worker
+
+```ts
+import { createMentionable } from '@mentionable/tracker/cloudflare'
+
+const mentionable = createMentionable({
+  apiKey: 'mtbl_xxxxx',
+})
+
+export default {
+  async fetch(request, env, ctx) {
+    // your existing logic
+    mentionable.track(request, ctx)
+
+    return fetch(request)
+  },
+}
+```
+
+## What Gets Tracked
+
+The tracker sends minimal, non-sensitive data to the Mentionable API:
+
+| Field    | Description                  |
+| -------- | ---------------------------- |
+| `ua`     | User-Agent header            |
+| `path`   | Request path (e.g. `/pricing`) |
+| `host`   | Hostname                     |
+| `method` | HTTP method                  |
+| `ip`     | Client IP (for bot identification) |
+| `status` | Response status code         |
+| `t`      | Timestamp                    |
+
+No cookies, no request bodies, no personal data.
+
+## Configuration
+
+```ts
+withMentionable({
+  // Required
+  apiKey: 'mtbl_xxxxx',
+})
+```
+
+## How It Works
+
+1. The middleware intercepts incoming requests
+2. It forwards the User-Agent + request metadata to the Mentionable API (fire-and-forget, zero latency impact)
+3. Mentionable detects and classifies the bot server-side
+4. Data appears in your dashboard within seconds
+
+The detection logic lives on Mentionable's servers, not in the package. When new AI bots emerge, they're detected automatically without any update on your end.
+
+## Performance
+
+The tracker is designed to have zero impact on your site:
+
+- **Non-blocking**: requests are sent with fire-and-forget (no `await`)
+- **Cloudflare**: uses `ctx.waitUntil()` so the response is never delayed
+- **Next.js**: runs on the Edge Runtime, adds < 1ms overhead
+- **Tiny payload**: ~200 bytes per event
+
+## Get Your API Key
+
+1. Sign up at [mentionable.com](https://mentionable.io)
+2. Go to Settings > API
+3. Copy your API key (starts with `mtbl_`)
+
+## Links
+
+- [Mentionable Dashboard](https://mentionable.io)
+
+## License
+
+MIT

package/dist/chunk-JBLZVSYS.js

@@ -0,0 +1,115 @@
+// src/core.ts
+var API_URL = "https://api.mentionable.io/v1/track";
+var BOTS_URL = "https://api.mentionable.io/v1/bots";
+var REFRESH_INTERVAL = 24 * 60 * 60 * 1e3;
+var FALLBACK_BOTS = [
+  // OpenAI
+  "GPTBot",
+  "ChatGPT-User",
+  "OAI-SearchBot",
+  // Anthropic
+  "ClaudeBot",
+  "anthropic-ai",
+  // Google
+  "Google-Extended",
+  "GoogleOther",
+  // Perplexity
+  "PerplexityBot",
+  // Meta
+  "Meta-ExternalAgent",
+  "FacebookBot",
+  // Apple
+  "Applebot-Extended",
+  // Amazon
+  "Amazonbot",
+  // ByteDance
+  "Bytespider",
+  // Common Crawl
+  "CCBot",
+  // Cohere
+  "cohere-ai",
+  // Diffbot
+  "Diffbot",
+  // You.com
+  "YouBot",
+  // AI2
+  "AI2Bot",
+  // xAI
+  "Grok",
+  // Huawei / Asiabot
+  "PetalBot",
+  // Webz.io
+  "Omgilibot",
+  // Imagesift
+  "ImagesiftBot",
+  // Timpi
+  "Timpibot"
+];
+function createBotMatcher() {
+  let patterns = FALLBACK_BOTS;
+  let lastFetch = 0;
+  let pending = null;
+  async function fetchPatterns(apiKey) {
+    try {
+      const res = await fetch(BOTS_URL, {
+        headers: { Authorization: `Bearer ${apiKey}` }
+      });
+      if (res.ok) {
+        const data = await res.json();
+        if (Array.isArray(data.patterns)) {
+          patterns = data.patterns;
+        }
+      }
+    } catch {
+    }
+    lastFetch = Date.now();
+  }
+  return {
+    /**
+     * Kick off a background refresh if the list is stale (>24 h) or never fetched.
+     * Returns the pending promise when a fetch was started, `undefined` otherwise.
+     */
+    refresh(apiKey) {
+      if (lastFetch > 0 && Date.now() - lastFetch < REFRESH_INTERVAL) return;
+      if (pending) return pending;
+      pending = fetchPatterns(apiKey).finally(() => {
+        pending = null;
+      });
+      return pending;
+    },
+    /** Case-insensitive substring match against the current pattern list. */
+    isBot(ua) {
+      const lower = ua.toLowerCase();
+      return patterns.some((p) => lower.includes(p.toLowerCase()));
+    }
+  };
+}
+function buildPayload(options) {
+  return {
+    ua: options.ua,
+    path: options.path,
+    host: options.host,
+    method: options.method,
+    ip: options.ip,
+    status: options.status ?? 200,
+    t: Date.now()
+  };
+}
+function sendEvent(event, apiKey) {
+  return fetch(API_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`
+    },
+    body: JSON.stringify(event)
+  }).then(() => {
+  }).catch(() => {
+  });
+}
+
+export {
+  createBotMatcher,
+  buildPayload,
+  sendEvent
+};

package/dist/cloudflare.d.ts

@@ -0,0 +1,47 @@
+import { M as MentionableConfig } from './core-CfJZm23p.js';
+
+interface CloudflareConfig extends MentionableConfig {
+    /** Set to `true` to read the API key from the `MENTIONABLE_API_KEY` env var at runtime. */
+    env?: boolean;
+}
+interface CloudflareEnv {
+    MENTIONABLE_API_KEY?: string;
+    [key: string]: unknown;
+}
+interface CloudflareExecutionContext {
+    waitUntil(promise: Promise<unknown>): void;
+    passThroughOnException(): void;
+}
+/**
+ * Standalone Worker handler that proxies requests to the origin and tracks AI bot crawls.
+ *
+ * ```ts
+ * import { withMentionable } from '@mentionable/tracker/cloudflare'
+ *
+ * export default withMentionable({
+ *   apiKey: 'mtbl_xxxxx',
+ * })
+ * ```
+ */
+declare function withMentionable(config: CloudflareConfig): {
+    fetch(request: Request, env: CloudflareEnv, ctx: CloudflareExecutionContext): Promise<Response>;
+};
+/**
+ * Composable tracker for use inside an existing Cloudflare Worker.
+ *
+ * ```ts
+ * const mentionable = createMentionable({ apiKey: 'mtbl_xxxxx' })
+ *
+ * export default {
+ *   async fetch(request, env, ctx) {
+ *     mentionable.track(request, ctx)
+ *     return fetch(request)
+ *   },
+ * }
+ * ```
+ */
+declare function createMentionable(config: MentionableConfig): {
+    track(request: Request, ctx: CloudflareExecutionContext): void;
+};
+
+export { type CloudflareConfig, MentionableConfig, createMentionable, withMentionable };

package/dist/cloudflare.js

@@ -0,0 +1,54 @@
+import {
+  buildPayload,
+  createBotMatcher,
+  sendEvent
+} from "./chunk-JBLZVSYS.js";
+
+// src/cloudflare.ts
+function extractPayload(request, status) {
+  return buildPayload({
+    ua: request.headers.get("user-agent") || "",
+    path: new URL(request.url).pathname,
+    host: request.headers.get("host") || "",
+    method: request.method,
+    ip: request.headers.get("cf-connecting-ip") || request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() || "",
+    status
+  });
+}
+function resolveApiKey(config, env) {
+  return config.env && env.MENTIONABLE_API_KEY || config.apiKey;
+}
+function withMentionable(config) {
+  const bots = createBotMatcher();
+  return {
+    async fetch(request, env, ctx) {
+      const apiKey = resolveApiKey(config, env);
+      const refreshing = bots.refresh(apiKey);
+      if (refreshing) ctx.waitUntil(refreshing);
+      const response = await fetch(request);
+      const ua = request.headers.get("user-agent") || "";
+      if (bots.isBot(ua)) {
+        const event = extractPayload(request, response.status);
+        ctx.waitUntil(sendEvent(event, apiKey));
+      }
+      return response;
+    }
+  };
+}
+function createMentionable(config) {
+  const bots = createBotMatcher();
+  return {
+    track(request, ctx) {
+      const refreshing = bots.refresh(config.apiKey);
+      if (refreshing) ctx.waitUntil(refreshing);
+      const ua = request.headers.get("user-agent") || "";
+      if (!bots.isBot(ua)) return;
+      const event = extractPayload(request);
+      ctx.waitUntil(sendEvent(event, config.apiKey));
+    }
+  };
+}
+export {
+  createMentionable,
+  withMentionable
+};

package/dist/core-CfJZm23p.d.ts

@@ -0,0 +1,5 @@
+interface MentionableConfig {
+    apiKey: string;
+}
+
+export type { MentionableConfig as M };

package/dist/next.d.ts

@@ -0,0 +1,33 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { M as MentionableConfig } from './core-CfJZm23p.js';
+
+/**
+ * Standalone middleware that tracks AI bot crawls.
+ *
+ * ```ts
+ * // middleware.ts
+ * import { withMentionable } from '@mentionable/tracker/next'
+ *
+ * export default withMentionable({
+ *   apiKey: process.env.MENTIONABLE_API_KEY!,
+ * })
+ * ```
+ */
+declare function withMentionable(config: MentionableConfig): (req: NextRequest) => NextResponse<unknown>;
+/**
+ * Composable tracker for use inside an existing middleware.
+ *
+ * ```ts
+ * const mentionable = createMentionable({ apiKey: process.env.MENTIONABLE_API_KEY! })
+ *
+ * export async function middleware(req) {
+ *   mentionable.track(req)
+ *   return NextResponse.next()
+ * }
+ * ```
+ */
+declare function createMentionable(config: MentionableConfig): {
+    track(req: NextRequest): void;
+};
+
+export { MentionableConfig, createMentionable, withMentionable };

package/dist/next.js

@@ -0,0 +1,44 @@
+import {
+  buildPayload,
+  createBotMatcher,
+  sendEvent
+} from "./chunk-JBLZVSYS.js";
+
+// src/next.ts
+import { NextResponse } from "next/server";
+function extractPayload(req) {
+  return buildPayload({
+    ua: req.headers.get("user-agent") || "",
+    path: new URL(req.url).pathname,
+    host: req.headers.get("host") || "",
+    method: req.method,
+    ip: req.headers.get("x-forwarded-for")?.split(",")[0]?.trim() || ""
+  });
+}
+function withMentionable(config) {
+  const bots = createBotMatcher();
+  return function middleware(req) {
+    bots.refresh(config.apiKey);
+    const ua = req.headers.get("user-agent") || "";
+    if (!bots.isBot(ua)) return NextResponse.next();
+    const event = extractPayload(req);
+    sendEvent(event, config.apiKey);
+    return NextResponse.next();
+  };
+}
+function createMentionable(config) {
+  const bots = createBotMatcher();
+  return {
+    track(req) {
+      bots.refresh(config.apiKey);
+      const ua = req.headers.get("user-agent") || "";
+      if (!bots.isBot(ua)) return;
+      const event = extractPayload(req);
+      sendEvent(event, config.apiKey);
+    }
+  };
+}
+export {
+  createMentionable,
+  withMentionable
+};

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@mentionable/tracker",
+  "version": "0.1.0",
+  "description": "Track AI bot crawls on your website. Monitor when GPTBot, ClaudeBot, PerplexityBot, Grok and others crawl your pages.",
+  "type": "module",
+  "exports": {
+    "./next": {
+      "types": "./dist/next.d.ts",
+      "default": "./dist/next.js"
+    },
+    "./cloudflare": {
+      "types": "./dist/cloudflare.d.ts",
+      "default": "./dist/cloudflare.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "next": "^15.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.0.0"
+  },
+  "peerDependencies": {
+    "next": ">=13.0.0"
+  },
+  "peerDependenciesMeta": {
+    "next": {
+      "optional": true
+    }
+  },
+  "keywords": [
+    "ai",
+    "bot",
+    "crawler",
+    "tracking",
+    "seo",
+    "mentionable",
+    "gptbot",
+    "claudebot",
+    "perplexitybot"
+  ],
+  "license": "MIT",
+  "sideEffects": false
+}