@statlark/bots 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,99 @@
-# Temporary Holding Version
+# @statlark/bots
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+See which AI assistants, search engines and training crawlers read your site.
+
+Most crawlers request raw HTML and never run JavaScript, so a tracker script can't see them.
+This reports them from your server instead. It adds no delay to your pages: reports are sent
+in the background, time out after 2 seconds, and never throw or fail a request.
+
+Full guide: <https://statlark.com/docs/bot-traffic>
+
+## Install
+
+```sh
+npm install @statlark/bots
+```
+
+Create a bot key under **Settings → Bot tracking**, then set it on your server as
+`STATLARK_BOT_KEY`. It's a secret: never give it a `NEXT_PUBLIC_` or `VITE_` prefix, and never
+put it in browser code. With no key, the package warns once and does nothing.
+
+## Next.js
+
+In `proxy.ts` (`middleware.ts` before Next 16), first, before any login check:
+
+```ts
+import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
+import { trackBot } from "@statlark/bots/next";
+
+export function proxy(request: NextRequest, event: NextFetchEvent) {
+  trackBot(request, event);
+  return NextResponse.next();
+}
+```
+
+On Vercel it reads the address from `x-real-ip`. Anywhere else, pass `{ ip }` yourself, or the
+check can't run. `x-forwarded-for` is never used: outside Vercel its leftmost entry is
+client-supplied, so a scanner could forge a verified request.
+
+## Cloudflare Workers and Pages
+
+```ts
+import { trackBot } from "@statlark/bots/cloudflare";
+
+export default {
+  async fetch(request, env, ctx) {
+    trackBot(request, ctx, env);
+    return handle(request, env);
+  },
+};
+```
+
+Static files never reach a Worker by default, so set `assets.run_worker_first` in your Wrangler
+config, or existing pages and `robots.txt` go uncounted. On Pages, `_routes.json` must not exclude
+them.
+
+## Express
+
+```ts
+import express from "express";
+import { statlarkBots } from "@statlark/bots/express";
+
+const app = express();
+app.set("trust proxy", 1); // your exact number of proxies
+app.use(statlarkBots());
+```
+
+Behind a proxy, `req.ip` is only right if `trust proxy` is an **exact hop count**. Never set it to
+`true`: that makes `req.ip` the client-supplied leftmost entry.
+
+## Any other server
+
+`reportBotHit` works anywhere with `fetch`:
+
+```ts
+import { reportBotHit } from "@statlark/bots";
+
+void reportBotHit({ url: request.url, userAgent: request.headers.get("user-agent"), ip });
+```
+
+Or call the endpoint yourself:
+
+```sh
+curl -X POST https://collect.statlark.com/bots \
+  -H "Authorization: Bearer $STATLARK_BOT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"url":"https://example.com/pricing","user_agent":"GPTBot/1.1","ip":"132.196.86.10"}'
+```
+
+Filter by user agent first, so you don't send every visitor to us. The guide has the filter.
+
+## What's sent
+
+The page's origin and path (the query string and fragment are stripped first), the user agent,
+and, for crawlers with a published address list, the IP. The IP is used to check the crawler is
+genuine, then discarded. Statlark keeps the crawler's name, the path and a count.
+
+## License
+
+MIT

package/dist/cloudflare.d.ts

@@ -0,0 +1,8 @@
+import { type TrackOptions } from "./index.js";
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export declare function trackBot(request: Request, ctx: {
+    waitUntil(p: Promise<unknown>): void;
+}, env: {
+    STATLARK_BOT_KEY?: string;
+}, options?: TrackOptions): void;
+//# sourceMappingURL=cloudflare.d.ts.map

package/dist/cloudflare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../src/cloudflare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D,6EAA6E;AAC7E,wBAAgB,QAAQ,CACtB,OAAO,EAAE,OAAO,EAChB,GAAG,EAAE;IAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;CAAE,EAC7C,GAAG,EAAE;IAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAAE,EAClC,OAAO,CAAC,EAAE,YAAY,GACrB,IAAI,CAcN"}

package/dist/cloudflare.js

@@ -0,0 +1,16 @@
+import { reportBotHit } from "./index.js";
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export function trackBot(request, ctx, env, options) {
+    try {
+        const promise = reportBotHit({
+            url: request.url,
+            userAgent: request.headers.get("user-agent"),
+            ip: request.headers.get("cf-connecting-ip"),
+        }, { ...options, key: options?.key ?? env?.STATLARK_BOT_KEY });
+        ctx?.waitUntil?.(promise);
+    }
+    catch {
+        // Never fail the request.
+    }
+}
+//# sourceMappingURL=cloudflare.js.map

package/dist/cloudflare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.js","sourceRoot":"","sources":["../src/cloudflare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAE7D,6EAA6E;AAC7E,MAAM,UAAU,QAAQ,CACtB,OAAgB,EAChB,GAA6C,EAC7C,GAAkC,EAClC,OAAsB;IAEtB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAC1B;YACE,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YAC5C,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;SAC5C,EACD,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,gBAAgB,EAAE,CAC3D,CAAC;QACF,GAAG,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;AACH,CAAC"}

package/dist/express.d.ts

@@ -0,0 +1,9 @@
+import { type TrackOptions } from "./index.js";
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * `req.ip` is only right behind a proxy if `trust proxy` is set to an exact hop count. Never
+ * set it to `true`, which makes `req.ip` the client-supplied leftmost entry.
+ */
+export declare function statlarkBots(options?: TrackOptions): (req: any, res: any, next: () => void) => void;
+//# sourceMappingURL=express.d.ts.map

package/dist/express.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.d.ts","sourceRoot":"","sources":["../src/express.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,CAiBhD"}

package/dist/express.js

@@ -0,0 +1,24 @@
+import { reportBotHit } from "./index.js";
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * `req.ip` is only right behind a proxy if `trust proxy` is set to an exact hop count. Never
+ * set it to `true`, which makes `req.ip` the client-supplied leftmost entry.
+ */
+export function statlarkBots(options) {
+    return (req, _res, next) => {
+        try {
+            void reportBotHit({
+                url: `${req.protocol}://${req.get("host")}${req.originalUrl}`,
+                userAgent: req.get("user-agent"),
+                ip: req.ip,
+            }, options);
+        }
+        catch {
+            // Never fail the request.
+        }
+        // Outside the try: an error from the next handler is the app's, not ours to swallow.
+        next();
+    };
+}
+//# sourceMappingURL=express.js.map

package/dist/express.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.js","sourceRoot":"","sources":["../src/express.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAE7D;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAsB;IAEtB,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QACzB,IAAI,CAAC;YACH,KAAK,YAAY,CACf;gBACE,GAAG,EAAE,GAAG,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,WAAW,EAAE;gBAC7D,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBAChC,EAAE,EAAE,GAAG,CAAC,EAAE;aACX,EACD,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QACD,qFAAqF;QACrF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+export type BotHit = {
+    /** An absolute URL or a path. The query string and fragment are never sent. */
+    url: string;
+    userAgent: string | null | undefined;
+    /** The crawler's address, if you have a reliable one. Used for the check, then discarded. */
+    ip?: string | null;
+};
+export type TrackOptions = {
+    /** Defaults to the STATLARK_BOT_KEY environment variable. */
+    key?: string;
+    endpoint?: string;
+    onError?: (e: unknown) => void;
+};
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export declare function reportBotHit(hit: BotHit, options?: TrackOptions): Promise<void>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAgCH,MAAM,MAAM,MAAM,GAAG;IACnB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,6FAA6F;IAC7F,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,6DAA6D;IAC7D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;CAChC,CAAC;AAsCF;sDACsD;AACtD,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAyEzF"}

package/dist/index.js

@@ -0,0 +1,147 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+const ENDPOINT = "https://collect.statlark.com/bots";
+const TIMEOUT_MS = 2000;
+const AUTH_BACKOFF_MS = 600000;
+const MAX_STOP_MS = 3600000;
+const FAILURE_BACKOFF_MS = 60000;
+const MAX_FAILURES = 3;
+const MAX_IN_FLIGHT = 20;
+// Cheap first filter. Statlark's server decides what's really a crawler (docs/21 D12), so
+// this only has to skip ordinary browsers without a network call. Keep it broad: a crawler
+// this misses never reaches the server, so adding it to the list wouldn't be enough.
+const CANDIDATE = /bot\b|bot\/|crawl|spider|slurp|-user\b|-index\b|-training\b|-agent\b|amzn-|google-|googleother|meta-|bytespider|cohere|facebookexternalhit/i;
+const SKIP = /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wasm)$/i;
+// Crawlers with a published list: the only ones whose IP is worth sending (D19).
+const HAS_LIST = /chatgpt-user|oai-searchbot|oai-adsbot|gptbot|claude|perplexity|google-agent|google-cloudvertexbot|googleother|googlebot|applebot|duckduckbot|duckassistbot|mistralai-(?:user|index)|ccbot|ahrefsbot/i;
+// Stands in for the origin of a bare path, so `new URL` can parse one.
+const NO_ORIGIN = "https://statlark.invalid";
+// Per key, so one bad key on a multi-tenant host can't stop another tenant's.
+const state = new Map();
+let inFlight = 0;
+let warnedNoKey = false;
+function stateFor(key) {
+    let entry = state.get(key);
+    if (!entry) {
+        entry = { stoppedUntil: 0, failures: 0, warned: false };
+        state.set(key, entry);
+    }
+    return entry;
+}
+/** The URL to send and its pathname, or null if it isn't something we'd report. */
+function pageUrl(url) {
+    const parsed = new URL(url, NO_ORIGIN);
+    if (parsed.origin === "null")
+        return null; // mailto:, javascript: and the like
+    const send = parsed.origin === NO_ORIGIN ? parsed.pathname : parsed.origin + parsed.pathname;
+    return { send, pathname: parsed.pathname };
+}
+async function sha256Hex(text) {
+    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+    return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
+function warnOnce(entry, message) {
+    if (entry.warned)
+        return;
+    entry.warned = true;
+    console.warn(message);
+}
+function countFailure(entry) {
+    entry.failures += 1;
+    if (entry.failures >= MAX_FAILURES) {
+        entry.failures = 0;
+        entry.stoppedUntil = Date.now() + FAILURE_BACKOFF_MS;
+    }
+}
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export async function reportBotHit(hit, options = {}) {
+    let entry;
+    let counted = false;
+    try {
+        if (typeof window !== "undefined")
+            return; // never run in a browser
+        const userAgent = hit.userAgent;
+        if (!userAgent || !CANDIDATE.test(userAgent))
+            return;
+        const page = pageUrl(hit.url);
+        if (!page || SKIP.test(page.pathname))
+            return;
+        const key = options.key ?? (typeof process !== "undefined" ? process.env.STATLARK_BOT_KEY : undefined);
+        if (!key) {
+            if (!warnedNoKey) {
+                warnedNoKey = true;
+                console.warn("[statlark] STATLARK_BOT_KEY isn't set, so bot tracking is off.");
+            }
+            return;
+        }
+        entry = stateFor(key);
+        if (Date.now() < entry.stoppedUntil || inFlight >= MAX_IN_FLIGHT)
+            return;
+        inFlight += 1;
+        counted = true;
+        const body = { url: page.send, user_agent: userAgent };
+        if (hit.ip) {
+            if (HAS_LIST.test(userAgent))
+                body.ip = hit.ip;
+            else
+                body.source = await sha256Hex(`${key}:${hit.ip}`);
+        }
+        const res = await fetch(options.endpoint ?? ENDPOINT, {
+            method: "POST",
+            headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(TIMEOUT_MS),
+        });
+        if (res.status === 401 || res.status === 402 || res.status === 403) {
+            entry.stoppedUntil = Date.now() + AUTH_BACKOFF_MS;
+            warnOnce(entry, `[statlark] Bot tracking stopped: the server answered ${res.status}. Check your bot key and plan.`);
+        }
+        else if (res.status === 429) {
+            const seconds = Number(res.headers.get("retry-after"));
+            const wait = seconds > 0 ? seconds * 1000 : FAILURE_BACKOFF_MS;
+            entry.stoppedUntil = Date.now() + Math.min(wait, MAX_STOP_MS);
+        }
+        else if (res.status >= 500) {
+            countFailure(entry);
+        }
+        else {
+            entry.failures = 0;
+        }
+        if (res.status === 200) {
+            const reply = (await res.json());
+            if (reply.paused_until) {
+                // Capped, so an upgrade takes effect within the hour.
+                const until = Date.parse(reply.paused_until);
+                const cap = Date.now() + MAX_STOP_MS;
+                entry.stoppedUntil = Number.isNaN(until) ? cap : Math.min(until, cap);
+            }
+        }
+        else {
+            await res.body?.cancel();
+        }
+    }
+    catch (e) {
+        if (entry)
+            countFailure(entry);
+        try {
+            options.onError?.(e);
+        }
+        catch {
+            // A throwing onError stays here too.
+        }
+    }
+    finally {
+        if (counted)
+            inFlight -= 1;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AACrD,MAAM,UAAU,GAAG,IAAK,CAAC;AACzB,MAAM,eAAe,GAAG,MAAO,CAAC;AAChC,MAAM,WAAW,GAAG,OAAS,CAAC;AAC9B,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAClC,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,0FAA0F;AAC1F,2FAA2F;AAC3F,qFAAqF;AACrF,MAAM,SAAS,GACb,6IAA6I,CAAC;AAChJ,MAAM,IAAI,GACR,6FAA6F,CAAC;AAEhG,iFAAiF;AACjF,MAAM,QAAQ,GACZ,sMAAsM,CAAC;AAEzM,uEAAuE;AACvE,MAAM,SAAS,GAAG,0BAA0B,CAAC;AAI7C,8EAA8E;AAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;AAC1C,IAAI,QAAQ,GAAG,CAAC,CAAC;AACjB,IAAI,WAAW,GAAG,KAAK,CAAC;AAiBxB,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACxD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mFAAmF;AACnF,SAAS,OAAO,CAAC,GAAW;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,oCAAoC;IAC/E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC7F,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC7C,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,SAAS,QAAQ,CAAC,KAAe,EAAE,OAAe;IAChD,IAAI,KAAK,CAAC,MAAM;QAAE,OAAO;IACzB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;IACpB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACnC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IACpB,IAAI,KAAK,CAAC,QAAQ,IAAI,YAAY,EAAE,CAAC;QACnC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnB,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;IACvD,CAAC;AACH,CAAC;AAED;sDACsD;AACtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,UAAwB,EAAE;IACxE,IAAI,KAA2B,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,WAAW;YAAE,OAAO,CAAC,yBAAyB;QACpE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO;QACrD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO;QAE9C,MAAM,GAAG,GACP,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7F,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;YACjF,CAAC;YACD,OAAO;QACT,CAAC;QAED,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,IAAI,QAAQ,IAAI,aAAa;YAAE,OAAO;QAEzE,QAAQ,IAAI,CAAC,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;QAEf,MAAM,IAAI,GAA2B,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QAC/E,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;gBAAE,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;;gBAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/E,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC;SACxC,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACnE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;YAClD,QAAQ,CAAC,KAAK,EAAE,wDAAwD,GAAG,CAAC,MAAM,gCAAgC,CAAC,CAAC;QACtH,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC;YAC/D,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC7B,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA8B,CAAC;YAC9D,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;gBACvB,sDAAsD;gBACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;gBACrC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,QAAQ,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC"}

package/dist/next.d.ts

@@ -0,0 +1,20 @@
+import { type TrackOptions } from "./index.js";
+type Req = {
+    url: string;
+    headers: Headers;
+};
+type Evt = {
+    waitUntil(p: Promise<unknown>): void;
+};
+/**
+ * Call this first in `proxy.ts` (`middleware.ts` before Next 16), before any login check.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export declare function trackBot(request: Req, event: Evt, options?: TrackOptions & {
+    ip?: string | null;
+}): void;
+export {};
+//# sourceMappingURL=next.d.ts.map

package/dist/next.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.d.ts","sourceRoot":"","sources":["../src/next.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D,KAAK,GAAG,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC;AAC7C,KAAK,GAAG,GAAG;IAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;CAAE,CAAC;AAEpD;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,OAAO,EAAE,GAAG,EACZ,KAAK,EAAE,GAAG,EACV,OAAO,CAAC,EAAE,YAAY,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC9C,IAAI,CAcN"}

package/dist/next.js

@@ -0,0 +1,22 @@
+import { reportBotHit } from "./index.js";
+/**
+ * Call this first in `proxy.ts` (`middleware.ts` before Next 16), before any login check.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export function trackBot(request, event, options) {
+    // The whole body is guarded, and so is a missing or wrapped `event`. Only crawlers reach the
+    // work below, so a manual test wouldn't show a failure here.
+    try {
+        const onVercel = typeof process !== "undefined" && Boolean(process.env.VERCEL);
+        const ip = options?.ip ?? (onVercel ? request.headers.get("x-real-ip") : null);
+        const promise = reportBotHit({ url: request.url, userAgent: request.headers.get("user-agent"), ip }, options);
+        event?.waitUntil?.(promise);
+    }
+    catch {
+        // Never fail the request.
+    }
+}
+//# sourceMappingURL=next.js.map

package/dist/next.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.js","sourceRoot":"","sources":["../src/next.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAK7D;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,OAAY,EACZ,KAAU,EACV,OAA+C;IAE/C,6FAA6F;IAC7F,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/E,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,YAAY,CAC1B,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,EACtE,OAAO,CACR,CAAC;QACF,KAAK,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,55 @@
 {
   "name": "@statlark/bots",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Statlark bot tracking: reports which AI assistants, search engines and training crawlers request your pages, from your server. Adapters for Next.js, Cloudflare and Express. No runtime dependencies.",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./next": {
+      "types": "./dist/next.d.ts",
+      "default": "./dist/next.js"
+    },
+    "./cloudflare": {
+      "types": "./dist/cloudflare.d.ts",
+      "default": "./dist/cloudflare.js"
+    },
+    "./express": {
+      "types": "./dist/express.d.ts",
+      "default": "./dist/express.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": ["dist", "src", "README.md"],
+  "publishConfig": {
+    "access": "public"
+  },
+  "sideEffects": false,
+  "engines": {
+    "node": ">=20.19"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "tsc"
+  },
+  "keywords": [
+    "statlark",
+    "analytics",
+    "bots",
+    "crawlers",
+    "ai-crawlers",
+    "gptbot",
+    "nextjs",
+    "cloudflare-workers",
+    "express"
+  ],
+  "devDependencies": {
+    "typescript": "^5.5.4"
+  }
+}

package/src/cloudflare.ts

@@ -0,0 +1,23 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export function trackBot(
+  request: Request,
+  ctx: { waitUntil(p: Promise<unknown>): void },
+  env: { STATLARK_BOT_KEY?: string },
+  options?: TrackOptions,
+): void {
+  try {
+    const promise = reportBotHit(
+      {
+        url: request.url,
+        userAgent: request.headers.get("user-agent"),
+        ip: request.headers.get("cf-connecting-ip"),
+      },
+      { ...options, key: options?.key ?? env?.STATLARK_BOT_KEY },
+    );
+    ctx?.waitUntil?.(promise);
+  } catch {
+    // Never fail the request.
+  }
+}

package/src/env.d.ts

@@ -0,0 +1,4 @@
+// The package has `"types": []` and no dependency on @types/node, and it also runs on Workers,
+// where `process` may not exist. So `process` is declared possibly undefined: every read has to
+// sit behind a `typeof process !== "undefined"` guard, and the compiler holds that line.
+declare const process: { env: Record<string, string | undefined> } | undefined;

package/src/express.ts

@@ -0,0 +1,28 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * `req.ip` is only right behind a proxy if `trust proxy` is set to an exact hop count. Never
+ * set it to `true`, which makes `req.ip` the client-supplied leftmost entry.
+ */
+export function statlarkBots(
+  options?: TrackOptions,
+): (req: any, res: any, next: () => void) => void {
+  return (req, _res, next) => {
+    try {
+      void reportBotHit(
+        {
+          url: `${req.protocol}://${req.get("host")}${req.originalUrl}`,
+          userAgent: req.get("user-agent"),
+          ip: req.ip,
+        },
+        options,
+      );
+    } catch {
+      // Never fail the request.
+    }
+    // Outside the try: an error from the next handler is the app's, not ours to swallow.
+    next();
+  };
+}

package/src/index.ts

@@ -0,0 +1,168 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+
+const ENDPOINT = "https://collect.statlark.com/bots";
+const TIMEOUT_MS = 2_000;
+const AUTH_BACKOFF_MS = 600_000;
+const MAX_STOP_MS = 3_600_000;
+const FAILURE_BACKOFF_MS = 60_000;
+const MAX_FAILURES = 3;
+const MAX_IN_FLIGHT = 20;
+
+// Cheap first filter. Statlark's server decides what's really a crawler (docs/21 D12), so
+// this only has to skip ordinary browsers without a network call. Keep it broad: a crawler
+// this misses never reaches the server, so adding it to the list wouldn't be enough.
+const CANDIDATE =
+  /bot\b|bot\/|crawl|spider|slurp|-user\b|-index\b|-training\b|-agent\b|amzn-|google-|googleother|meta-|bytespider|cohere|facebookexternalhit/i;
+const SKIP =
+  /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wasm)$/i;
+
+// Crawlers with a published list: the only ones whose IP is worth sending (D19).
+const HAS_LIST =
+  /chatgpt-user|oai-searchbot|oai-adsbot|gptbot|claude|perplexity|google-agent|google-cloudvertexbot|googleother|googlebot|applebot|duckduckbot|duckassistbot|mistralai-(?:user|index)|ccbot|ahrefsbot/i;
+
+// Stands in for the origin of a bare path, so `new URL` can parse one.
+const NO_ORIGIN = "https://statlark.invalid";
+
+type KeyState = { stoppedUntil: number; failures: number; warned: boolean };
+
+// Per key, so one bad key on a multi-tenant host can't stop another tenant's.
+const state = new Map<string, KeyState>();
+let inFlight = 0;
+let warnedNoKey = false;
+
+export type BotHit = {
+  /** An absolute URL or a path. The query string and fragment are never sent. */
+  url: string;
+  userAgent: string | null | undefined;
+  /** The crawler's address, if you have a reliable one. Used for the check, then discarded. */
+  ip?: string | null;
+};
+
+export type TrackOptions = {
+  /** Defaults to the STATLARK_BOT_KEY environment variable. */
+  key?: string;
+  endpoint?: string;
+  onError?: (e: unknown) => void;
+};
+
+function stateFor(key: string): KeyState {
+  let entry = state.get(key);
+  if (!entry) {
+    entry = { stoppedUntil: 0, failures: 0, warned: false };
+    state.set(key, entry);
+  }
+  return entry;
+}
+
+/** The URL to send and its pathname, or null if it isn't something we'd report. */
+function pageUrl(url: string): { send: string; pathname: string } | null {
+  const parsed = new URL(url, NO_ORIGIN);
+  if (parsed.origin === "null") return null; // mailto:, javascript: and the like
+  const send = parsed.origin === NO_ORIGIN ? parsed.pathname : parsed.origin + parsed.pathname;
+  return { send, pathname: parsed.pathname };
+}
+
+async function sha256Hex(text: string): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+  return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+function warnOnce(entry: KeyState, message: string): void {
+  if (entry.warned) return;
+  entry.warned = true;
+  console.warn(message);
+}
+
+function countFailure(entry: KeyState): void {
+  entry.failures += 1;
+  if (entry.failures >= MAX_FAILURES) {
+    entry.failures = 0;
+    entry.stoppedUntil = Date.now() + FAILURE_BACKOFF_MS;
+  }
+}
+
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export async function reportBotHit(hit: BotHit, options: TrackOptions = {}): Promise<void> {
+  let entry: KeyState | undefined;
+  let counted = false;
+  try {
+    if (typeof window !== "undefined") return; // never run in a browser
+    const userAgent = hit.userAgent;
+    if (!userAgent || !CANDIDATE.test(userAgent)) return;
+    const page = pageUrl(hit.url);
+    if (!page || SKIP.test(page.pathname)) return;
+
+    const key =
+      options.key ?? (typeof process !== "undefined" ? process.env.STATLARK_BOT_KEY : undefined);
+    if (!key) {
+      if (!warnedNoKey) {
+        warnedNoKey = true;
+        console.warn("[statlark] STATLARK_BOT_KEY isn't set, so bot tracking is off.");
+      }
+      return;
+    }
+
+    entry = stateFor(key);
+    if (Date.now() < entry.stoppedUntil || inFlight >= MAX_IN_FLIGHT) return;
+
+    inFlight += 1;
+    counted = true;
+
+    const body: Record<string, string> = { url: page.send, user_agent: userAgent };
+    if (hit.ip) {
+      if (HAS_LIST.test(userAgent)) body.ip = hit.ip;
+      else body.source = await sha256Hex(`${key}:${hit.ip}`);
+    }
+
+    const res = await fetch(options.endpoint ?? ENDPOINT, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(TIMEOUT_MS),
+    });
+
+    if (res.status === 401 || res.status === 402 || res.status === 403) {
+      entry.stoppedUntil = Date.now() + AUTH_BACKOFF_MS;
+      warnOnce(entry, `[statlark] Bot tracking stopped: the server answered ${res.status}. Check your bot key and plan.`);
+    } else if (res.status === 429) {
+      const seconds = Number(res.headers.get("retry-after"));
+      const wait = seconds > 0 ? seconds * 1000 : FAILURE_BACKOFF_MS;
+      entry.stoppedUntil = Date.now() + Math.min(wait, MAX_STOP_MS);
+    } else if (res.status >= 500) {
+      countFailure(entry);
+    } else {
+      entry.failures = 0;
+    }
+
+    if (res.status === 200) {
+      const reply = (await res.json()) as { paused_until?: string };
+      if (reply.paused_until) {
+        // Capped, so an upgrade takes effect within the hour.
+        const until = Date.parse(reply.paused_until);
+        const cap = Date.now() + MAX_STOP_MS;
+        entry.stoppedUntil = Number.isNaN(until) ? cap : Math.min(until, cap);
+      }
+    } else {
+      await res.body?.cancel();
+    }
+  } catch (e) {
+    if (entry) countFailure(entry);
+    try {
+      options.onError?.(e);
+    } catch {
+      // A throwing onError stays here too.
+    }
+  } finally {
+    if (counted) inFlight -= 1;
+  }
+}

package/src/next.ts

@@ -0,0 +1,31 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+type Req = { url: string; headers: Headers };
+type Evt = { waitUntil(p: Promise<unknown>): void };
+
+/**
+ * Call this first in `proxy.ts` (`middleware.ts` before Next 16), before any login check.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export function trackBot(
+  request: Req,
+  event: Evt,
+  options?: TrackOptions & { ip?: string | null },
+): void {
+  // The whole body is guarded, and so is a missing or wrapped `event`. Only crawlers reach the
+  // work below, so a manual test wouldn't show a failure here.
+  try {
+    const onVercel = typeof process !== "undefined" && Boolean(process.env.VERCEL);
+    const ip = options?.ip ?? (onVercel ? request.headers.get("x-real-ip") : null);
+    const promise = reportBotHit(
+      { url: request.url, userAgent: request.headers.get("user-agent"), ip },
+      options,
+    );
+    event?.waitUntil?.(promise);
+  } catch {
+    // Never fail the request.
+  }
+}