@statlark/bots 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,131 @@
-# Temporary Holding Version
+# @statlark/bots
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+See which AI assistants, search engines and training crawlers read your site.
+
+Most crawlers request raw HTML and never run JavaScript, so a tracker script can't see them.
+This reports them from your server instead. Reports go out in the background, so they don't
+hold up your responses. They time out after 2 seconds and never throw or fail a request.
+
+Full guide: <https://statlark.com/docs/bot-traffic>
+
+## Install
+
+```sh
+npm install @statlark/bots
+```
+
+Create a bot key under **Settings → Bot tracking**, then set it on your server as
+`STATLARK_BOT_KEY`. It's a secret: never give it a `NEXT_PUBLIC_` or `VITE_` prefix, and never
+put it in browser code. With no key, the package warns once and does nothing.
+
+## Next.js
+
+In `proxy.ts`, first, before any login check. Before Next 16 the file is `middleware.ts` and
+the function is `middleware`:
+
+```ts
+import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
+import { trackBot } from "@statlark/bots/next";
+
+export function proxy(request: NextRequest, event: NextFetchEvent) {
+  trackBot(request, event);
+  return NextResponse.next();
+}
+```
+
+On Vercel it reads the address from `x-real-ip`. Anywhere else, pass `{ ip }` yourself, or the
+check can't run. `x-forwarded-for` is never used: outside Vercel its leftmost entry is
+client-supplied, so a scanner could forge a verified request.
+
+## Cloudflare Workers and Pages
+
+```ts
+import { trackBot } from "@statlark/bots/cloudflare";
+
+export default {
+  async fetch(request, env, ctx) {
+    trackBot(request, ctx, env);
+    return handle(request, env);
+  },
+};
+```
+
+Static files never reach a Worker by default, so existing pages and `robots.txt` go uncounted
+unless the Worker runs first. Set `run_worker_first` to patterns that leave your asset folders
+out:
+
+```toml
+[assets]
+directory = "./public"
+binding = "ASSETS"
+run_worker_first = ["/*", "!/assets/*"] # use your own asset folders
+```
+
+Every request that reaches the Worker counts towards Cloudflare's limits. On the Workers Free
+plan that's 100,000 a day, and past it those URLs fail until midnight UTC, your homepage
+included. Excluding your asset folders keeps those requests free. Exclude folders, not file
+types: a path that matches an exclusion never reaches your Worker. On Pages, requests that
+reach a Function count the same way, so keep asset folders out in `_routes.json`, but not your
+pages or `robots.txt`.
+
+## Express
+
+```ts
+import express from "express";
+import { statlarkBots } from "@statlark/bots/express";
+
+const app = express();
+app.use(statlarkBots());
+```
+
+The IP comes from `req.ip`, so Express's `trust proxy` setting decides it, for your whole app:
+
+- Nothing in front of Node: don't set `trust proxy`. With any value, `req.ip` comes from the
+  `X-Forwarded-For` the client sent.
+- Behind proxies you control that add `X-Forwarded-For`: set it to exactly that many hops, for
+  example `app.set("trust proxy", 1)` for one. Never `true`, and never more hops than there are.
+- Already set in your app: leave it as it is.
+- Not sure: leave it out. Real crawlers then read as fake, which is safe, and the guide's
+  troubleshooting section has the fix.
+
+This package is ESM-only. `require()` works on Node ^20.19 or >=22.12; on older versions use
+`import` or `await import()`. A CommonJS TypeScript project needs `"module": "nodenext"` on
+TypeScript 5.8 or later.
+
+## Any other server
+
+`reportBotHit` works anywhere with `fetch`:
+
+```ts
+import { reportBotHit } from "@statlark/bots";
+
+void reportBotHit({ url: request.url, userAgent: request.headers.get("user-agent"), ip });
+```
+
+Or call the endpoint yourself:
+
+```sh
+curl -X POST https://collect.statlark.com/bots \
+  -H "Authorization: Bearer $STATLARK_BOT_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"url":"https://example.com/pricing","user_agent":"GPTBot/1.1","ip":"132.196.86.10"}'
+```
+
+Filter by user agent first, so you don't send every visitor to us. The guide has the filter.
+
+## What's sent
+
+The page's origin and path (the query string and fragment are stripped first) and the user
+agent. When an IP is available, a crawler with a published address list also sends it, so
+Statlark can check the crawler is genuine. Any other request that looks like a crawler sends
+`source` instead: a SHA-256 hash of your bot key and the address, with an IPv6 address cut to
+its /64 first, used only to rate-limit crawlers that can't be checked. Neither is written to
+Statlark's database. The hash is a pseudonym, not anonymous: anyone holding your key could work
+out an IPv4 address from it. Statlark keeps the crawler's name, the path and a count.
+
+Framework paths such as `/_next/` and `/api/` aren't sent, and neither is a user agent the
+server has just said it doesn't recognise, for an hour.
+
+## License
+
+MIT

package/dist/cloudflare.d.ts

@@ -0,0 +1,8 @@
+import { type TrackOptions } from "./index.js";
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export declare function trackBot(request: Request, ctx: {
+    waitUntil(p: Promise<unknown>): void;
+}, env: {
+    STATLARK_BOT_KEY?: string;
+}, options?: TrackOptions): void;
+//# sourceMappingURL=cloudflare.d.ts.map

package/dist/cloudflare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.d.ts","sourceRoot":"","sources":["../src/cloudflare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D,6EAA6E;AAC7E,wBAAgB,QAAQ,CACtB,OAAO,EAAE,OAAO,EAChB,GAAG,EAAE;IAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;CAAE,EAC7C,GAAG,EAAE;IAAE,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAAE,EAClC,OAAO,CAAC,EAAE,YAAY,GACrB,IAAI,CAcN"}

package/dist/cloudflare.js

@@ -0,0 +1,16 @@
+import { reportBotHit } from "./index.js";
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export function trackBot(request, ctx, env, options) {
+    try {
+        const promise = reportBotHit({
+            url: request.url,
+            userAgent: request.headers.get("user-agent"),
+            ip: request.headers.get("cf-connecting-ip"),
+        }, { ...options, key: options?.key ?? env?.STATLARK_BOT_KEY });
+        ctx?.waitUntil?.(promise);
+    }
+    catch {
+        // Never fail the request.
+    }
+}
+//# sourceMappingURL=cloudflare.js.map

package/dist/cloudflare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudflare.js","sourceRoot":"","sources":["../src/cloudflare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAE7D,6EAA6E;AAC7E,MAAM,UAAU,QAAQ,CACtB,OAAgB,EAChB,GAA6C,EAC7C,GAAkC,EAClC,OAAsB;IAEtB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAC1B;YACE,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YAC5C,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;SAC5C,EACD,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,gBAAgB,EAAE,CAC3D,CAAC;QACF,GAAG,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;AACH,CAAC"}

package/dist/express.d.ts

@@ -0,0 +1,10 @@
+import { type TrackOptions } from "./index.js";
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * The IP comes from `req.ip`, so `trust proxy` decides it. With nothing in front of Node, don't
+ * set it. Behind proxies you control, set the exact number of hops. Never set it to `true`, or
+ * to more hops than there are: `req.ip` then comes from a header the client can write.
+ */
+export declare function statlarkBots(options?: TrackOptions): (req: any, res: any, next: () => void) => void;
+//# sourceMappingURL=express.d.ts.map

package/dist/express.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.d.ts","sourceRoot":"","sources":["../src/express.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,OAAO,CAAC,EAAE,YAAY,GACrB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK,IAAI,CAiBhD"}

package/dist/express.js

@@ -0,0 +1,25 @@
+import { reportBotHit } from "./index.js";
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * The IP comes from `req.ip`, so `trust proxy` decides it. With nothing in front of Node, don't
+ * set it. Behind proxies you control, set the exact number of hops. Never set it to `true`, or
+ * to more hops than there are: `req.ip` then comes from a header the client can write.
+ */
+export function statlarkBots(options) {
+    return (req, _res, next) => {
+        try {
+            void reportBotHit({
+                url: `${req.protocol}://${req.get("host")}${req.originalUrl}`,
+                userAgent: req.get("user-agent"),
+                ip: req.ip,
+            }, options);
+        }
+        catch {
+            // Never fail the request.
+        }
+        // Outside the try: an error from the next handler is the app's, not ours to swallow.
+        next();
+    };
+}
+//# sourceMappingURL=express.js.map

package/dist/express.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.js","sourceRoot":"","sources":["../src/express.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAE7D;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAsB;IAEtB,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QACzB,IAAI,CAAC;YACH,KAAK,YAAY,CACf;gBACE,GAAG,EAAE,GAAG,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,WAAW,EAAE;gBAC7D,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBAChC,EAAE,EAAE,GAAG,CAAC,EAAE;aACX,EACD,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QACD,qFAAqF;QACrF,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+export type BotHit = {
+    /** An absolute URL or a path. The query string and fragment are never sent. */
+    url: string;
+    userAgent: string | null | undefined;
+    /**
+     * The crawler's address, if you have a reliable one. Sent for crawlers with a published list,
+     * to check them. For other crawlers only a hash of it, keyed with your bot key, is sent, for a
+     * rate limit. Neither is written to Statlark's database.
+     */
+    ip?: string | null;
+};
+export type TrackOptions = {
+    /** Defaults to the STATLARK_BOT_KEY environment variable. */
+    key?: string;
+    endpoint?: string;
+    onError?: (e: unknown) => void;
+};
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export declare function reportBotHit(hit: BotHit, options?: TrackOptions): Promise<void>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA4CH,MAAM,MAAM,MAAM,GAAG;IACnB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,6DAA6D;IAC7D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;CAChC,CAAC;AAoEF;sDACsD;AACtD,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6FzF"}

package/dist/index.js

@@ -0,0 +1,213 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+const ENDPOINT = "https://collect.statlark.com/bots";
+const TIMEOUT_MS = 2000;
+const AUTH_BACKOFF_MS = 600000;
+const MAX_STOP_MS = 3600000;
+const FAILURE_BACKOFF_MS = 60000;
+const MAX_FAILURES = 3;
+const MAX_IN_FLIGHT = 20;
+// The server refuses a longer user agent, so don't send one.
+const MAX_USER_AGENT = 512;
+// After the server says a user agent isn't a crawler it knows, skip it for an hour. The hour
+// keeps a crawler newly added to the server's list from waiting on an upgrade (docs/21 D12).
+const UNKNOWN_TTL_MS = 3600000;
+const MAX_UNKNOWN = 500;
+// Cheap first filter. Statlark's server decides what's really a crawler (docs/21 D12), so
+// this only has to skip ordinary browsers without a network call. Keep it broad: a crawler
+// this misses never reaches the server, so adding it to the list wouldn't be enough.
+const CANDIDATE = /bot\b|bot\/|crawl|spider|slurp|-user\b|-index\b|-training\b|-agent\b|amzn-|google-|googleother|meta-|bytespider|cohere|facebookexternalhit/i;
+const SKIP = /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wasm)$/i;
+// The server drops these paths, so don't spend a request on them.
+const FRAMEWORK = /^\/(?:_next|_vercel|api|\.well-known)(?:\/|$)/;
+// A key is printable ASCII. Anything else, such as a pasted line break, can't go in a header.
+const BAD_KEY = /[^\x21-\x7e]/;
+// Crawlers with a published list: the only ones whose IP is worth sending (D19).
+const HAS_LIST = /chatgpt-user|oai-searchbot|oai-adsbot|gptbot|claude|perplexity|google-agent|google-cloudvertexbot|googleother|googlebot|applebot|duckduckbot|duckassistbot|mistralai-(?:user|index)|ccbot|ahrefsbot/i;
+// Stands in for the origin of a bare path, so `new URL` can parse one.
+const NO_ORIGIN = "https://statlark.invalid";
+// Per key, so one bad key on a multi-tenant host can't stop another tenant's.
+const state = new Map();
+// User agents the server answered unknown_crawler for, with when to try again.
+const unknown = new Map();
+let inFlight = 0;
+let warnedNoKey = false;
+function stateFor(key) {
+    let entry = state.get(key);
+    if (!entry) {
+        entry = { stoppedUntil: 0, failures: 0, warned: false };
+        state.set(key, entry);
+    }
+    return entry;
+}
+/** The URL to send and its pathname, or null if it isn't something we'd report. */
+function pageUrl(url) {
+    const parsed = new URL(url, NO_ORIGIN);
+    if (parsed.origin === "null")
+        return null; // mailto:, javascript: and the like
+    const send = parsed.origin === NO_ORIGIN ? parsed.pathname : parsed.origin + parsed.pathname;
+    return { send, pathname: parsed.pathname };
+}
+async function sha256Hex(text) {
+    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+    return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
+/**
+ * What a rate limit should treat as one source: an IPv4 address, or an IPv6 /64, since one host
+ * can hold a whole /64. A mapped IPv4 address stays whole, so dual-stack servers don't put every
+ * IPv4 client in one bucket.
+ */
+function sourceAddress(ip) {
+    const address = ip.split("%")[0].toLowerCase();
+    if (!address.includes(":") || address.includes("."))
+        return address;
+    const [head, tail] = address.split("::");
+    if (address.split("::").length > 2)
+        return address;
+    const front = head ? head.split(":") : [];
+    const back = tail === undefined ? [] : tail ? tail.split(":") : [];
+    const groups = tail === undefined ? front : [...front, ...Array(8 - front.length - back.length).fill("0"), ...back];
+    if (groups.length !== 8 || !groups.every((g) => /^[0-9a-f]{1,4}$/.test(g)))
+        return address;
+    return `${groups.slice(0, 4).map((g) => g.replace(/^0+(?=.)/, "")).join(":")}::/64`;
+}
+/** Statlark's own errors are JSON with a string `error`. A 401, 402 or 403 without that came
+ *  from something on the way, so it says nothing about the key or the plan. */
+async function isStatlarkError(res) {
+    if (!res.headers.get("content-type")?.includes("application/json"))
+        return false;
+    try {
+        const reply = (await res.json());
+        return typeof reply?.error === "string";
+    }
+    catch {
+        return false;
+    }
+}
+function warnOnce(entry, message) {
+    if (entry.warned)
+        return;
+    entry.warned = true;
+    console.warn(message);
+}
+function countFailure(entry) {
+    entry.failures += 1;
+    if (entry.failures >= MAX_FAILURES) {
+        entry.failures = 0;
+        entry.stoppedUntil = Date.now() + FAILURE_BACKOFF_MS;
+    }
+}
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export async function reportBotHit(hit, options = {}) {
+    let entry;
+    let counted = false;
+    try {
+        if (typeof window !== "undefined")
+            return; // never run in a browser
+        const userAgent = hit.userAgent;
+        if (!userAgent || userAgent.length > MAX_USER_AGENT || !CANDIDATE.test(userAgent))
+            return;
+        const retryAt = unknown.get(userAgent);
+        if (retryAt !== undefined) {
+            if (Date.now() < retryAt)
+                return;
+            unknown.delete(userAgent);
+        }
+        const page = pageUrl(hit.url);
+        if (!page || SKIP.test(page.pathname) || FRAMEWORK.test(page.pathname))
+            return;
+        // Trimmed, so a key pasted with a trailing line break still works.
+        const key = (options.key ?? (typeof process !== "undefined" ? process.env.STATLARK_BOT_KEY : undefined))?.trim();
+        if (!key) {
+            if (!warnedNoKey) {
+                warnedNoKey = true;
+                console.warn("[statlark] STATLARK_BOT_KEY isn't set, so bot tracking is off.");
+            }
+            return;
+        }
+        entry = stateFor(key);
+        if (BAD_KEY.test(key)) {
+            warnOnce(entry, "[statlark] Bot tracking is off: the bot key contains a character a key can't have, such as a line break. Copy it again.");
+            return;
+        }
+        if (Date.now() < entry.stoppedUntil || inFlight >= MAX_IN_FLIGHT)
+            return;
+        inFlight += 1;
+        counted = true;
+        const body = { url: page.send, user_agent: userAgent };
+        if (hit.ip) {
+            if (HAS_LIST.test(userAgent))
+                body.ip = hit.ip;
+            else
+                body.source = await sha256Hex(`${key}:${sourceAddress(hit.ip)}`);
+        }
+        const res = await fetch(options.endpoint ?? ENDPOINT, {
+            method: "POST",
+            headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(TIMEOUT_MS),
+        });
+        if (res.status === 401 || res.status === 402 || res.status === 403) {
+            if (await isStatlarkError(res)) {
+                entry.stoppedUntil = Date.now() + AUTH_BACKOFF_MS;
+                warnOnce(entry, `[statlark] Bot tracking stopped: the server answered ${res.status}. Check your bot key and plan.`);
+            }
+            else {
+                // A layer on the way, such as a firewall, answered. Not a verdict on the key.
+                countFailure(entry);
+            }
+        }
+        else if (res.status === 429) {
+            const seconds = Number(res.headers.get("retry-after"));
+            const wait = seconds > 0 ? seconds * 1000 : FAILURE_BACKOFF_MS;
+            entry.stoppedUntil = Date.now() + Math.min(wait, MAX_STOP_MS);
+        }
+        else if (res.status >= 500) {
+            countFailure(entry);
+        }
+        else {
+            entry.failures = 0;
+        }
+        if (res.status === 200) {
+            const reply = (await res.json());
+            if (reply.reason === "unknown_crawler") {
+                if (unknown.size >= MAX_UNKNOWN)
+                    unknown.delete(unknown.keys().next().value);
+                unknown.set(userAgent, Date.now() + UNKNOWN_TTL_MS);
+            }
+            if (reply.paused_until) {
+                // Capped, so an upgrade takes effect within the hour.
+                const until = Date.parse(reply.paused_until);
+                const cap = Date.now() + MAX_STOP_MS;
+                entry.stoppedUntil = Number.isNaN(until) ? cap : Math.min(until, cap);
+            }
+        }
+        else if (!res.bodyUsed) {
+            await res.body?.cancel();
+        }
+    }
+    catch (e) {
+        if (entry)
+            countFailure(entry);
+        try {
+            options.onError?.(e);
+        }
+        catch {
+            // A throwing onError stays here too.
+        }
+    }
+    finally {
+        if (counted)
+            inFlight -= 1;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,QAAQ,GAAG,mCAAmC,CAAC;AACrD,MAAM,UAAU,GAAG,IAAK,CAAC;AACzB,MAAM,eAAe,GAAG,MAAO,CAAC;AAChC,MAAM,WAAW,GAAG,OAAS,CAAC;AAC9B,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAClC,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,6DAA6D;AAC7D,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAM,cAAc,GAAG,OAAS,CAAC;AACjC,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,0FAA0F;AAC1F,2FAA2F;AAC3F,qFAAqF;AACrF,MAAM,SAAS,GACb,6IAA6I,CAAC;AAChJ,MAAM,IAAI,GACR,6FAA6F,CAAC;AAChG,kEAAkE;AAClE,MAAM,SAAS,GAAG,+CAA+C,CAAC;AAClE,8FAA8F;AAC9F,MAAM,OAAO,GAAG,cAAc,CAAC;AAE/B,iFAAiF;AACjF,MAAM,QAAQ,GACZ,sMAAsM,CAAC;AAEzM,uEAAuE;AACvE,MAAM,SAAS,GAAG,0BAA0B,CAAC;AAI7C,8EAA8E;AAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;AAC1C,+EAA+E;AAC/E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;AAC1C,IAAI,QAAQ,GAAG,CAAC,CAAC;AACjB,IAAI,WAAW,GAAG,KAAK,CAAC;AAqBxB,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACxD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mFAAmF;AACnF,SAAS,OAAO,CAAC,GAAW;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,oCAAoC;IAC/E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC7F,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC7C,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAY;IACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/C,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACpE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,MAAM,MAAM,GACV,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IACvG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,OAAO,CAAC;IAC3F,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;AACtF,CAAC;AAED;+EAC+E;AAC/E,KAAK,UAAU,eAAe,CAAC,GAAa;IAC1C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA+B,CAAC;QAC/D,OAAO,OAAO,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAe,EAAE,OAAe;IAChD,IAAI,KAAK,CAAC,MAAM;QAAE,OAAO;IACzB,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;IACpB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACnC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IACpB,IAAI,KAAK,CAAC,QAAQ,IAAI,YAAY,EAAE,CAAC;QACnC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QACnB,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;IACvD,CAAC;AACH,CAAC;AAED;sDACsD;AACtD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,UAAwB,EAAE;IACxE,IAAI,KAA2B,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,OAAO,MAAM,KAAK,WAAW;YAAE,OAAO,CAAC,yBAAyB;QACpE,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,OAAO;QAC1F,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;gBAAE,OAAO;YACjC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO;QAE/E,mEAAmE;QACnE,MAAM,GAAG,GAAG,CACV,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CAAC,CAC3F,EAAE,IAAI,EAAE,CAAC;QACV,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;YACjF,CAAC;YACD,OAAO;QACT,CAAC;QAED,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,KAAK,EAAE,yHAAyH,CAAC,CAAC;YAC3I,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,YAAY,IAAI,QAAQ,IAAI,aAAa;YAAE,OAAO;QAEzE,QAAQ,IAAI,CAAC,CAAC;QACd,OAAO,GAAG,IAAI,CAAC;QAEf,MAAM,IAAI,GAA2B,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QAC/E,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACX,IAAI,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;gBAAE,IAAI,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;;gBAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,GAAG,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,QAAQ,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/E,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC;SACxC,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACnE,IAAI,MAAM,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;gBAClD,QAAQ,CAAC,KAAK,EAAE,wDAAwD,GAAG,CAAC,MAAM,gCAAgC,CAAC,CAAC;YACtH,CAAC;iBAAM,CAAC;gBACN,8EAA8E;gBAC9E,YAAY,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC;YAC/D,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC7B,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA+C,CAAC;YAC/E,IAAI,KAAK,CAAC,MAAM,KAAK,iBAAiB,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,IAAI,IAAI,WAAW;oBAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAe,CAAC,CAAC;gBACvF,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;gBACvB,sDAAsD;gBACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;gBACrC,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,OAAO;YAAE,QAAQ,IAAI,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC"}

package/dist/next.d.ts

@@ -0,0 +1,21 @@
+import { type TrackOptions } from "./index.js";
+type Req = {
+    url: string;
+    headers: Headers;
+};
+type Evt = {
+    waitUntil(p: Promise<unknown>): void;
+};
+/**
+ * Call this first in `proxy.ts`, before any login check. Before Next 16 the file is
+ * `middleware.ts` and the function is `middleware`.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export declare function trackBot(request: Req, event: Evt, options?: TrackOptions & {
+    ip?: string | null;
+}): void;
+export {};
+//# sourceMappingURL=next.d.ts.map

package/dist/next.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.d.ts","sourceRoot":"","sources":["../src/next.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7D,KAAK,GAAG,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC;AAC7C,KAAK,GAAG,GAAG;IAAE,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;CAAE,CAAC;AAEpD;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CACtB,OAAO,EAAE,GAAG,EACZ,KAAK,EAAE,GAAG,EACV,OAAO,CAAC,EAAE,YAAY,GAAG;IAAE,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC9C,IAAI,CAcN"}

package/dist/next.js

@@ -0,0 +1,23 @@
+import { reportBotHit } from "./index.js";
+/**
+ * Call this first in `proxy.ts`, before any login check. Before Next 16 the file is
+ * `middleware.ts` and the function is `middleware`.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export function trackBot(request, event, options) {
+    // The whole body is guarded, and so is a missing or wrapped `event`. Only crawlers reach the
+    // work below, so a manual test wouldn't show a failure here.
+    try {
+        const onVercel = typeof process !== "undefined" && Boolean(process.env.VERCEL);
+        const ip = options?.ip ?? (onVercel ? request.headers.get("x-real-ip") : null);
+        const promise = reportBotHit({ url: request.url, userAgent: request.headers.get("user-agent"), ip }, options);
+        event?.waitUntil?.(promise);
+    }
+    catch {
+        // Never fail the request.
+    }
+}
+//# sourceMappingURL=next.js.map

package/dist/next.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.js","sourceRoot":"","sources":["../src/next.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAqB,MAAM,YAAY,CAAC;AAK7D;;;;;;;GAOG;AACH,MAAM,UAAU,QAAQ,CACtB,OAAY,EACZ,KAAU,EACV,OAA+C;IAE/C,6FAA6F;IAC7F,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/E,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,YAAY,CAC1B,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,EACtE,OAAO,CACR,CAAC;QACF,KAAK,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,62 @@
 {
   "name": "@statlark/bots",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Statlark bot tracking: reports which AI assistants, search engines and training crawlers request your pages, from your server. Adapters for Next.js, Cloudflare and Express. No runtime dependencies.",
+  "license": "MIT",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "next": ["dist/next.d.ts"],
+      "cloudflare": ["dist/cloudflare.d.ts"],
+      "express": ["dist/express.d.ts"]
+    }
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./next": {
+      "types": "./dist/next.d.ts",
+      "default": "./dist/next.js"
+    },
+    "./cloudflare": {
+      "types": "./dist/cloudflare.d.ts",
+      "default": "./dist/cloudflare.js"
+    },
+    "./express": {
+      "types": "./dist/express.d.ts",
+      "default": "./dist/express.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": ["dist", "src", "README.md"],
+  "publishConfig": {
+    "access": "public"
+  },
+  "sideEffects": false,
+  "engines": {
+    "node": "^20.19.0 || >=22.12.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "tsc"
+  },
+  "keywords": [
+    "statlark",
+    "analytics",
+    "bots",
+    "crawlers",
+    "ai-crawlers",
+    "gptbot",
+    "nextjs",
+    "cloudflare-workers",
+    "express"
+  ],
+  "devDependencies": {
+    "typescript": "^5.5.4"
+  }
+}

package/src/cloudflare.ts

@@ -0,0 +1,23 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+/** Call this at the top of your Worker's `fetch`, or in a Pages Function. */
+export function trackBot(
+  request: Request,
+  ctx: { waitUntil(p: Promise<unknown>): void },
+  env: { STATLARK_BOT_KEY?: string },
+  options?: TrackOptions,
+): void {
+  try {
+    const promise = reportBotHit(
+      {
+        url: request.url,
+        userAgent: request.headers.get("user-agent"),
+        ip: request.headers.get("cf-connecting-ip"),
+      },
+      { ...options, key: options?.key ?? env?.STATLARK_BOT_KEY },
+    );
+    ctx?.waitUntil?.(promise);
+  } catch {
+    // Never fail the request.
+  }
+}

package/src/env.d.ts

@@ -0,0 +1,4 @@
+// The package has `"types": []` and no dependency on @types/node, and it also runs on Workers,
+// where `process` may not exist. So `process` is declared possibly undefined: every read has to
+// sit behind a `typeof process !== "undefined"` guard, and the compiler holds that line.
+declare const process: { env: Record<string, string | undefined> } | undefined;

package/src/express.ts

@@ -0,0 +1,29 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+/**
+ * `app.use(statlarkBots())`. Fire and forget: the request carries on immediately.
+ *
+ * The IP comes from `req.ip`, so `trust proxy` decides it. With nothing in front of Node, don't
+ * set it. Behind proxies you control, set the exact number of hops. Never set it to `true`, or
+ * to more hops than there are: `req.ip` then comes from a header the client can write.
+ */
+export function statlarkBots(
+  options?: TrackOptions,
+): (req: any, res: any, next: () => void) => void {
+  return (req, _res, next) => {
+    try {
+      void reportBotHit(
+        {
+          url: `${req.protocol}://${req.get("host")}${req.originalUrl}`,
+          userAgent: req.get("user-agent"),
+          ip: req.ip,
+        },
+        options,
+      );
+    } catch {
+      // Never fail the request.
+    }
+    // Outside the try: an error from the next handler is the app's, not ours to swallow.
+    next();
+  };
+}

package/src/index.ts

@@ -0,0 +1,234 @@
+/**
+ * @statlark/bots: tells Statlark when a crawler requests a page.
+ *
+ * Most AI crawlers never run JavaScript, so this has to happen on the server. It's a thin
+ * reporter: this file makes a cheap first guess at what's a crawler, and Statlark's server
+ * decides what really is one (docs/21 D12).
+ *
+ * THE RULE: this runs in the middle of someone else's site. It never throws, never rejects,
+ * never retries, and gives up after 2 seconds. A failure here must never reach a request.
+ */
+
+const ENDPOINT = "https://collect.statlark.com/bots";
+const TIMEOUT_MS = 2_000;
+const AUTH_BACKOFF_MS = 600_000;
+const MAX_STOP_MS = 3_600_000;
+const FAILURE_BACKOFF_MS = 60_000;
+const MAX_FAILURES = 3;
+const MAX_IN_FLIGHT = 20;
+// The server refuses a longer user agent, so don't send one.
+const MAX_USER_AGENT = 512;
+// After the server says a user agent isn't a crawler it knows, skip it for an hour. The hour
+// keeps a crawler newly added to the server's list from waiting on an upgrade (docs/21 D12).
+const UNKNOWN_TTL_MS = 3_600_000;
+const MAX_UNKNOWN = 500;
+
+// Cheap first filter. Statlark's server decides what's really a crawler (docs/21 D12), so
+// this only has to skip ordinary browsers without a network call. Keep it broad: a crawler
+// this misses never reaches the server, so adding it to the list wouldn't be enough.
+const CANDIDATE =
+  /bot\b|bot\/|crawl|spider|slurp|-user\b|-index\b|-training\b|-agent\b|amzn-|google-|googleother|meta-|bytespider|cohere|facebookexternalhit/i;
+const SKIP =
+  /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wasm)$/i;
+// The server drops these paths, so don't spend a request on them.
+const FRAMEWORK = /^\/(?:_next|_vercel|api|\.well-known)(?:\/|$)/;
+// A key is printable ASCII. Anything else, such as a pasted line break, can't go in a header.
+const BAD_KEY = /[^\x21-\x7e]/;
+
+// Crawlers with a published list: the only ones whose IP is worth sending (D19).
+const HAS_LIST =
+  /chatgpt-user|oai-searchbot|oai-adsbot|gptbot|claude|perplexity|google-agent|google-cloudvertexbot|googleother|googlebot|applebot|duckduckbot|duckassistbot|mistralai-(?:user|index)|ccbot|ahrefsbot/i;
+
+// Stands in for the origin of a bare path, so `new URL` can parse one.
+const NO_ORIGIN = "https://statlark.invalid";
+
+type KeyState = { stoppedUntil: number; failures: number; warned: boolean };
+
+// Per key, so one bad key on a multi-tenant host can't stop another tenant's.
+const state = new Map<string, KeyState>();
+// User agents the server answered unknown_crawler for, with when to try again.
+const unknown = new Map<string, number>();
+let inFlight = 0;
+let warnedNoKey = false;
+
+export type BotHit = {
+  /** An absolute URL or a path. The query string and fragment are never sent. */
+  url: string;
+  userAgent: string | null | undefined;
+  /**
+   * The crawler's address, if you have a reliable one. Sent for crawlers with a published list,
+   * to check them. For other crawlers only a hash of it, keyed with your bot key, is sent, for a
+   * rate limit. Neither is written to Statlark's database.
+   */
+  ip?: string | null;
+};
+
+export type TrackOptions = {
+  /** Defaults to the STATLARK_BOT_KEY environment variable. */
+  key?: string;
+  endpoint?: string;
+  onError?: (e: unknown) => void;
+};
+
+function stateFor(key: string): KeyState {
+  let entry = state.get(key);
+  if (!entry) {
+    entry = { stoppedUntil: 0, failures: 0, warned: false };
+    state.set(key, entry);
+  }
+  return entry;
+}
+
+/** The URL to send and its pathname, or null if it isn't something we'd report. */
+function pageUrl(url: string): { send: string; pathname: string } | null {
+  const parsed = new URL(url, NO_ORIGIN);
+  if (parsed.origin === "null") return null; // mailto:, javascript: and the like
+  const send = parsed.origin === NO_ORIGIN ? parsed.pathname : parsed.origin + parsed.pathname;
+  return { send, pathname: parsed.pathname };
+}
+
+async function sha256Hex(text: string): Promise<string> {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+  return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+/**
+ * What a rate limit should treat as one source: an IPv4 address, or an IPv6 /64, since one host
+ * can hold a whole /64. A mapped IPv4 address stays whole, so dual-stack servers don't put every
+ * IPv4 client in one bucket.
+ */
+function sourceAddress(ip: string): string {
+  const address = ip.split("%")[0].toLowerCase();
+  if (!address.includes(":") || address.includes(".")) return address;
+  const [head, tail] = address.split("::");
+  if (address.split("::").length > 2) return address;
+  const front = head ? head.split(":") : [];
+  const back = tail === undefined ? [] : tail ? tail.split(":") : [];
+  const groups =
+    tail === undefined ? front : [...front, ...Array(8 - front.length - back.length).fill("0"), ...back];
+  if (groups.length !== 8 || !groups.every((g) => /^[0-9a-f]{1,4}$/.test(g))) return address;
+  return `${groups.slice(0, 4).map((g) => g.replace(/^0+(?=.)/, "")).join(":")}::/64`;
+}
+
+/** Statlark's own errors are JSON with a string `error`. A 401, 402 or 403 without that came
+ *  from something on the way, so it says nothing about the key or the plan. */
+async function isStatlarkError(res: Response): Promise<boolean> {
+  if (!res.headers.get("content-type")?.includes("application/json")) return false;
+  try {
+    const reply = (await res.json()) as { error?: unknown } | null;
+    return typeof reply?.error === "string";
+  } catch {
+    return false;
+  }
+}
+
+function warnOnce(entry: KeyState, message: string): void {
+  if (entry.warned) return;
+  entry.warned = true;
+  console.warn(message);
+}
+
+function countFailure(entry: KeyState): void {
+  entry.failures += 1;
+  if (entry.failures >= MAX_FAILURES) {
+    entry.failures = 0;
+    entry.stoppedUntil = Date.now() + FAILURE_BACKOFF_MS;
+  }
+}
+
+/** Reports a crawler's request. Never throws, never rejects, and always returns a promise,
+ *  so it's safe to hand straight to any waitUntil. */
+export async function reportBotHit(hit: BotHit, options: TrackOptions = {}): Promise<void> {
+  let entry: KeyState | undefined;
+  let counted = false;
+  try {
+    if (typeof window !== "undefined") return; // never run in a browser
+    const userAgent = hit.userAgent;
+    if (!userAgent || userAgent.length > MAX_USER_AGENT || !CANDIDATE.test(userAgent)) return;
+    const retryAt = unknown.get(userAgent);
+    if (retryAt !== undefined) {
+      if (Date.now() < retryAt) return;
+      unknown.delete(userAgent);
+    }
+    const page = pageUrl(hit.url);
+    if (!page || SKIP.test(page.pathname) || FRAMEWORK.test(page.pathname)) return;
+
+    // Trimmed, so a key pasted with a trailing line break still works.
+    const key = (
+      options.key ?? (typeof process !== "undefined" ? process.env.STATLARK_BOT_KEY : undefined)
+    )?.trim();
+    if (!key) {
+      if (!warnedNoKey) {
+        warnedNoKey = true;
+        console.warn("[statlark] STATLARK_BOT_KEY isn't set, so bot tracking is off.");
+      }
+      return;
+    }
+
+    entry = stateFor(key);
+    if (BAD_KEY.test(key)) {
+      warnOnce(entry, "[statlark] Bot tracking is off: the bot key contains a character a key can't have, such as a line break. Copy it again.");
+      return;
+    }
+    if (Date.now() < entry.stoppedUntil || inFlight >= MAX_IN_FLIGHT) return;
+
+    inFlight += 1;
+    counted = true;
+
+    const body: Record<string, string> = { url: page.send, user_agent: userAgent };
+    if (hit.ip) {
+      if (HAS_LIST.test(userAgent)) body.ip = hit.ip;
+      else body.source = await sha256Hex(`${key}:${sourceAddress(hit.ip)}`);
+    }
+
+    const res = await fetch(options.endpoint ?? ENDPOINT, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(TIMEOUT_MS),
+    });
+
+    if (res.status === 401 || res.status === 402 || res.status === 403) {
+      if (await isStatlarkError(res)) {
+        entry.stoppedUntil = Date.now() + AUTH_BACKOFF_MS;
+        warnOnce(entry, `[statlark] Bot tracking stopped: the server answered ${res.status}. Check your bot key and plan.`);
+      } else {
+        // A layer on the way, such as a firewall, answered. Not a verdict on the key.
+        countFailure(entry);
+      }
+    } else if (res.status === 429) {
+      const seconds = Number(res.headers.get("retry-after"));
+      const wait = seconds > 0 ? seconds * 1000 : FAILURE_BACKOFF_MS;
+      entry.stoppedUntil = Date.now() + Math.min(wait, MAX_STOP_MS);
+    } else if (res.status >= 500) {
+      countFailure(entry);
+    } else {
+      entry.failures = 0;
+    }
+
+    if (res.status === 200) {
+      const reply = (await res.json()) as { paused_until?: string; reason?: string };
+      if (reply.reason === "unknown_crawler") {
+        if (unknown.size >= MAX_UNKNOWN) unknown.delete(unknown.keys().next().value as string);
+        unknown.set(userAgent, Date.now() + UNKNOWN_TTL_MS);
+      }
+      if (reply.paused_until) {
+        // Capped, so an upgrade takes effect within the hour.
+        const until = Date.parse(reply.paused_until);
+        const cap = Date.now() + MAX_STOP_MS;
+        entry.stoppedUntil = Number.isNaN(until) ? cap : Math.min(until, cap);
+      }
+    } else if (!res.bodyUsed) {
+      await res.body?.cancel();
+    }
+  } catch (e) {
+    if (entry) countFailure(entry);
+    try {
+      options.onError?.(e);
+    } catch {
+      // A throwing onError stays here too.
+    }
+  } finally {
+    if (counted) inFlight -= 1;
+  }
+}

package/src/next.ts

@@ -0,0 +1,32 @@
+import { reportBotHit, type TrackOptions } from "./index.js";
+
+type Req = { url: string; headers: Headers };
+type Evt = { waitUntil(p: Promise<unknown>): void };
+
+/**
+ * Call this first in `proxy.ts`, before any login check. Before Next 16 the file is
+ * `middleware.ts` and the function is `middleware`.
+ *
+ * Only `x-real-ip` on Vercel is trusted. `x-forwarded-for` is never used: outside Vercel its
+ * leftmost entry is client-supplied, so a scanner could forge a verified request. Behind
+ * anything else, pass `options.ip` yourself.
+ */
+export function trackBot(
+  request: Req,
+  event: Evt,
+  options?: TrackOptions & { ip?: string | null },
+): void {
+  // The whole body is guarded, and so is a missing or wrapped `event`. Only crawlers reach the
+  // work below, so a manual test wouldn't show a failure here.
+  try {
+    const onVercel = typeof process !== "undefined" && Boolean(process.env.VERCEL);
+    const ip = options?.ip ?? (onVercel ? request.headers.get("x-real-ip") : null);
+    const promise = reportBotHit(
+      { url: request.url, userAgent: request.headers.get("user-agent"), ip },
+      options,
+    );
+    event?.waitUntil?.(promise);
+  } catch {
+    // Never fail the request.
+  }
+}