@traceten/ai-crawl 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,184 @@
+/**
+ * Public types for @traceten/ai-crawl.
+ *
+ * The package is a PRE-FILTER only: it decides "is this plausibly an AI
+ * crawler worth reporting" locally, and sends a minimal payload. Provider
+ * classification, category, verification and confidence are all decided
+ * server-side, so the crawler list can change without customers upgrading.
+ * The local token list is a cost filter, not the source of truth.
+ */
+/**
+ * The four-category taxonomy Traceten uses server-side. Used locally ONLY to
+ * apply the `disable*` opt-outs — the authoritative category is assigned
+ * server-side.
+ */
+export type CrawlerCategory = "answer_fetch" | "search_index" | "training" | "ai_crawler";
+/** Result of the local two-tier user-agent match. */
+export interface CrawlerMatch {
+    /** Exact agent token that matched (tier 1), or `null` for a tier-2 provider-alias match. */
+    readonly agent: string | null;
+    /** Coarse provider slug (e.g. `"openai"`). */
+    readonly provider: string;
+    /** Local category guess — used only for the `disable*` opt-outs. */
+    readonly category: CrawlerCategory;
+    /** `"exact"` = tier-1 agent token; `"provider"` = tier-2 coarse alias. */
+    readonly tier: "exact" | "provider";
+}
+/** User-supplied configuration. Validate with {@link import("./config.js").defineAiCrawlConfig}. */
+export interface AiCrawlConfig {
+    /**
+     * The site's public site key — the `ttid_…` value shown on the dashboard's
+     * install page (`data-site`). Case-sensitive. Sent as `site_id` on the
+     * wire. Required.
+     */
+    siteId: string;
+    /**
+     * Server-side auth token (`tt_bot_...`). REQUIRED — `POST /v1/ai-crawls`
+     * rejects unauthenticated reports. Unlike the browser snippet, a
+     * server-side package can hold a real secret; requiring it removes the
+     * cost-attack and data-poisoning surface. Missing/malformed tokens fail
+     * loudly at construction rather than silently sending requests that 401.
+     */
+    authToken: string;
+    /** Ingestion endpoint. Default: `https://ingest.traceten.com/v1/ai-crawls`. */
+    endpoint?: string;
+    /** HTTP methods to consider. Default `["GET", "HEAD"]`. Replaces the default. */
+    allowedMethods?: readonly string[];
+    /** Extra path prefixes to deny, ADDED to the built-in list (never replaces it). */
+    extraDenyPathPrefixes?: readonly string[];
+    /** Extra file extensions to deny (with or without leading dot), ADDED to the built-in list. */
+    extraDenyExtensions?: readonly string[];
+    /** Skip reporting `answer_fetch` crawlers (ChatGPT-User, Claude-User, …). */
+    disableAnswerFetch?: boolean;
+    /** Skip reporting `search_index` crawlers (OAI-SearchBot, PerplexityBot, …). */
+    disableSearchCrawlers?: boolean;
+    /** Skip reporting `training` crawlers (GPTBot, ClaudeBot, CCBot, …). */
+    disableTrainingCrawlers?: boolean;
+    /** Skip reporting `ai_crawler` (uncategorised) crawlers and tier-2 provider-alias matches. */
+    disableOtherCrawlers?: boolean;
+    /**
+     * Trust `x-forwarded-for` when resolving the crawler IP. Default `false`:
+     * XFF is attacker-controlled unless a proxy you operate strips/appends it,
+     * so it is never trusted blindly. Set `true` only when your app sits behind
+     * proxies you control, and set {@link proxyDepth} to how many of them
+     * append to XFF.
+     */
+    trustProxy?: boolean;
+    /**
+     * Trust the `cf-connecting-ip` header. Default `false`: on an origin NOT
+     * behind Cloudflare, any client can forge this header, and a forged
+     * vendor-range IP paired with a vendor UA is exactly the spoof the
+     * server-side verification exists to catch. The two Cloudflare adapters
+     * enable it automatically (the platform strips and rewrites the header
+     * there). Set it manually only when your origin genuinely sits behind
+     * Cloudflare (e.g. Express behind orange-cloud DNS).
+     */
+    trustCfConnectingIp?: boolean;
+    /**
+     * Number of trusted reverse proxies that append to `x-forwarded-for`.
+     * Default `1`. With N trusted proxies the crawler IP is the Nth entry from
+     * the right (rightmost-untrusted-aware selection). Only used when
+     * {@link trustProxy} is `true`.
+     */
+    proxyDepth?: number;
+    /**
+     * Origin (e.g. `"https://example.com"`) used to rebuild the reported URL
+     * when the runtime sees an internal one (containers / reverse proxies that
+     * construct `request.url` from an internal host).
+     */
+    publicOrigin?: string;
+    /**
+     * Called when a report does not reach the endpoint: a non-2xx response, or
+     * a network/timeout failure.
+     *
+     * Delivery stays silent by contract. This never throws into your request
+     * path, nothing is retried, and the response is not surfaced any other way.
+     * Without it a rejected report is indistinguishable from a delivered one, so
+     * a misconfigured token or blocked egress reads as "no crawlers visited"
+     * indefinitely. Wire it to your logger in staging at minimum.
+     *
+     * Your callback is itself wrapped in try/catch. Throwing from it cannot
+     * break the host response.
+     */
+    onError?: (error: AiCrawlDeliveryError) => void;
+    /** Injectable fetch, for tests. Defaults to the global `fetch`. */
+    fetch?: typeof fetch;
+}
+/** Why a report failed to land. Passed to {@link AiCrawlConfig.onError}. */
+export interface AiCrawlDeliveryError {
+    /**
+     * `"http"` when the endpoint answered with a non-2xx status.
+     * `"network"` when the request never completed (DNS, TLS, timeout, abort).
+     */
+    kind: "http" | "network";
+    /** Status when `kind` is `"http"`. A 401/403 means the token or the origin. */
+    status?: number;
+    /** The thrown value when `kind` is `"network"`. */
+    cause?: unknown;
+    /** The endpoint the report was addressed to. */
+    endpoint: string;
+}
+/** Config after validation + defaulting. Produced by `defineAiCrawlConfig`. */
+export interface ResolvedAiCrawlConfig {
+    readonly siteId: string;
+    readonly authToken: string;
+    readonly endpoint: string;
+    readonly allowedMethods: readonly string[];
+    readonly denyPathPrefixes: readonly string[];
+    readonly denyExtensions: readonly string[];
+    readonly disableAnswerFetch: boolean;
+    readonly disableSearchCrawlers: boolean;
+    readonly disableTrainingCrawlers: boolean;
+    readonly disableOtherCrawlers: boolean;
+    readonly trustProxy: boolean;
+    readonly trustCfConnectingIp: boolean;
+    readonly proxyDepth: number;
+    readonly publicOrigin: string | undefined;
+    readonly onError: ((error: AiCrawlDeliveryError) => void) | undefined;
+    readonly fetch: typeof fetch | undefined;
+    /** Brand so per-request paths can tell a validated config from a raw one. */
+    readonly __resolved: true;
+}
+/**
+ * Wire payload for `POST /v1/ai-crawls`. snake_case on the wire, per
+ * Traceten's wire-format convention.
+ *
+ * `ip` is the CRAWLER's IP as observed by the customer's server — the only
+ * place it is observable, since the crawler's TCP connection terminates
+ * there. Traceten's edge verifies it against vendor-published ranges while
+ * it is in scope; retention is verdict-dependent and decided server-side.
+ * The field is omitted entirely when no trustworthy value is derivable —
+ * never guessed.
+ *
+ * ⚠️ `ip` is a CLAIM made by customer-controlled software, not network
+ * proof. The receiving service must verify it (CIDR / rDNS) before treating
+ * it as crawler infrastructure, and must never let an unverified claimed IP
+ * mint a verified/raw-retained row.
+ */
+export interface AiCrawlWirePayload {
+    site_id: string;
+    /** Full request URL (absolute where derivable). */
+    url: string;
+    method: string;
+    /** Response status where the adapter can observe it; omitted otherwise. */
+    status?: number;
+    user_agent: string;
+    /** Crawler IP. Omitted rather than wrong. */
+    ip?: string;
+    /** Epoch milliseconds at observation time. */
+    ts: number;
+}
+/**
+ * Runtime-agnostic view of a request, produced by each adapter.
+ * Header names are looked up case-insensitively by the adapter.
+ */
+export interface RequestFacts {
+    method: string;
+    /** Absolute URL if the runtime provides one, else a path like `/docs/x`. */
+    url: string;
+    /** Case-insensitive header lookup; returns `null` when absent. */
+    header: (name: string) => string | null;
+    /** Socket remote address, where the runtime exposes one (Node servers). */
+    socketAddr?: string | undefined;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,cAAc,GAAG,UAAU,GAAG,YAAY,CAAC;AAE1F,qDAAqD;AACrD,MAAM,WAAW,YAAY;IAC3B,4FAA4F;IAC5F,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,8CAA8C;IAC9C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oEAAoE;IACpE,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,0EAA0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,UAAU,CAAC;CACrC;AAED,oGAAoG;AACpG,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,mFAAmF;IACnF,qBAAqB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gFAAgF;IAChF,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,8FAA8F;IAC9F,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;;;;OAQG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAChD,mEAAmE;IACnE,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,4EAA4E;AAC5E,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,+EAA+E;AAC/E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC;IACxC,QAAQ,CAAC,uBAAuB,EAAE,OAAO,CAAC;IAC1C,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACtE,QAAQ,CAAC,KAAK,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IACzC,6EAA6E;IAC7E,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,EAAE,EAAE,MAAM,CAAC;CACZ;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,GAAG,EAAE,MAAM,CAAC;IACZ,kEAAkE;IAClE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACxC,2EAA2E;IAC3E,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACjC"}

package/dist/types.js

@@ -0,0 +1,11 @@
+/**
+ * Public types for @traceten/ai-crawl.
+ *
+ * The package is a PRE-FILTER only: it decides "is this plausibly an AI
+ * crawler worth reporting" locally, and sends a minimal payload. Provider
+ * classification, category, verification and confidence are all decided
+ * server-side, so the crawler list can change without customers upgrading.
+ * The local token list is a cost filter, not the source of truth.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG"}

package/package.json

@@ -0,0 +1,87 @@
+{
+  "name": "@traceten/ai-crawl",
+  "version": "0.1.0",
+  "private": false,
+  "description": "Traceten server-side AI crawler tracking — see the GPTBot/ClaudeBot/PerplexityBot crawls that browser JavaScript can never see. Five adapters, zero dependencies, never blocks a response.",
+  "keywords": [
+    "ai-crawler",
+    "gptbot",
+    "claudebot",
+    "perplexitybot",
+    "bot-detection",
+    "analytics",
+    "traceten"
+  ],
+  "license": "MIT",
+  "homepage": "https://docs.traceten.com/install/ai-crawler-tracking",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/traceten/ai-crawl.git"
+  },
+  "bugs": {
+    "url": "https://github.com/traceten/ai-crawl/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "sideEffects": false,
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./next": {
+      "types": "./dist/adapters/next.d.ts",
+      "import": "./dist/adapters/next.js",
+      "default": "./dist/adapters/next.js"
+    },
+    "./cloudflare-pages": {
+      "types": "./dist/adapters/cloudflare-pages.d.ts",
+      "import": "./dist/adapters/cloudflare-pages.js",
+      "default": "./dist/adapters/cloudflare-pages.js"
+    },
+    "./cloudflare-workers": {
+      "types": "./dist/adapters/cloudflare-workers.d.ts",
+      "import": "./dist/adapters/cloudflare-workers.js",
+      "default": "./dist/adapters/cloudflare-workers.js"
+    },
+    "./express": {
+      "types": "./dist/adapters/express.d.ts",
+      "import": "./dist/adapters/express.js",
+      "default": "./dist/adapters/express.js"
+    },
+    "./hono": {
+      "types": "./dist/adapters/hono.d.ts",
+      "import": "./dist/adapters/hono.js",
+      "default": "./dist/adapters/hono.js"
+    }
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "CHANGELOG.md",
+    "src",
+    "!src/**/*.test.ts"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.json && node build.mjs && ./check-size.sh",
+    "prepack": "npm run build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit --project tsconfig.json",
+    "lint": "echo 'no linter configured yet'"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "esbuild": "^0.24.0",
+    "typescript": "^5.7.0",
+    "vitest": "^2.0.0"
+  }
+}

package/src/adapters/cloudflare-pages.ts

@@ -0,0 +1,64 @@
+/**
+ * Cloudflare Pages Functions adapter — drop into `functions/_middleware.ts`.
+ *
+ * ```ts
+ * // functions/_middleware.ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { createAICrawlerPagesMiddleware } from "@traceten/ai-crawl/cloudflare-pages";
+ *
+ * export const onRequest = createAICrawlerPagesMiddleware(
+ *   defineAiCrawlConfig({
+ *     siteId: "…",           // or read from context.env inside your own wrapper
+ *     authToken: "tt_bot_…",
+ *   }),
+ * );
+ * ```
+ *
+ * Construction throws on invalid config (deploy-time failure); the returned
+ * middleware never throws and never delays the response — delivery goes
+ * through `context.waitUntil` and observes the REAL response status.
+ */
+
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts, type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+
+/** Structural subset of Pages Functions' `EventContext`. */
+export interface PagesContext {
+  request: FetchLikeRequest;
+  next(): Promise<{ status: number }> | { status: number };
+  waitUntil(promise: Promise<unknown>): void;
+}
+
+type PagesMiddleware = (context: PagesContext) => Promise<{ status: number }>;
+
+/**
+ * Build a Pages Functions middleware. THROWS at construction when the config
+ * is invalid — that is deploy time, exactly where a bad token should fail.
+ */
+export function createAICrawlerPagesMiddleware(
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): PagesMiddleware {
+  const resolved = isResolvedConfig(config) ? config : defineAiCrawlConfig(config);
+  // On Pages the platform strips and rewrites cf-connecting-ip, so it is
+  // trustworthy here (and only here / Workers) by default.
+  const cfg: ResolvedAiCrawlConfig = { ...resolved, trustCfConnectingIp: true };
+
+  return async (context: PagesContext) => {
+    // The customer's response comes first and is never touched. If next()
+    // itself throws, that error propagates untouched — swallowing it would
+    // change the site's behaviour.
+    const response = await context.next();
+    try {
+      trackFacts(
+        cfg,
+        factsFromFetchRequest(context.request),
+        typeof response?.status === "number" ? response.status : undefined,
+        (p) => context.waitUntil(p),
+      );
+    } catch {
+      /* silent by contract */
+    }
+    return response;
+  };
+}

package/src/adapters/cloudflare-workers.ts

@@ -0,0 +1,70 @@
+/**
+ * Cloudflare Workers adapter — wrap your fetch handler.
+ *
+ * For a customer already behind Cloudflare this is copy-paste with no
+ * application deploy, which is why this adapter matters disproportionately.
+ *
+ * ```ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { withAICrawlerTracking } from "@traceten/ai-crawl/cloudflare-workers";
+ *
+ * const config = defineAiCrawlConfig({ siteId: "…", authToken: "tt_bot_…" });
+ *
+ * export default {
+ *   fetch: withAICrawlerTracking(async (request, env, ctx) => {
+ *     return await handle(request);
+ *   }, config),
+ * };
+ * ```
+ *
+ * Construction throws on invalid config; the wrapper never throws its own
+ * errors, captures the REAL status code from the handler's response, and
+ * schedules delivery via `ctx.waitUntil`. Handler errors propagate untouched.
+ */
+
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts, type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+
+/** Structural subset of the Workers `ExecutionContext`. */
+export interface WorkersExecutionContext {
+  waitUntil(promise: Promise<unknown>): void;
+}
+
+type WorkersHandler<Env, Res extends { status: number }> = (
+  request: FetchLikeRequest,
+  env: Env,
+  ctx: WorkersExecutionContext,
+) => Promise<Res> | Res;
+
+/**
+ * Wrap a Workers fetch handler with AI crawler tracking. THROWS at
+ * construction (module scope — deploy time) when the config is invalid.
+ */
+export function withAICrawlerTracking<Env, Res extends { status: number }>(
+  handler: WorkersHandler<Env, Res>,
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): WorkersHandler<Env, Res> {
+  const resolved = isResolvedConfig(config) ? config : defineAiCrawlConfig(config);
+  // On Workers the platform strips and rewrites cf-connecting-ip, so it is
+  // trustworthy here (and only here / Pages) by default.
+  const cfg: ResolvedAiCrawlConfig = { ...resolved, trustCfConnectingIp: true };
+
+  return async (request, env, ctx) => {
+    // Handler errors propagate untouched — tracking must not change behaviour.
+    const response = await handler(request, env, ctx);
+    try {
+      trackFacts(
+        cfg,
+        factsFromFetchRequest(request),
+        typeof response?.status === "number" ? response.status : undefined,
+        ctx !== undefined && typeof ctx.waitUntil === "function"
+          ? (p) => ctx.waitUntil(p)
+          : undefined,
+      );
+    } catch {
+      /* silent by contract */
+    }
+    return response;
+  };
+}

package/src/adapters/express.ts

@@ -0,0 +1,113 @@
+/**
+ * Express adapter (also works for Connect-style and bare Node servers with a
+ * compatible `(req, res, next)` signature).
+ *
+ * ```ts
+ * import { createTracetenAICrawlerMiddleware } from "@traceten/ai-crawl/express";
+ *
+ * app.use(
+ *   createTracetenAICrawlerMiddleware({
+ *     siteId: process.env.TRACETEN_SITE_ID!,
+ *     authToken: process.env.TRACETEN_CRAWL_TOKEN!,
+ *   }),
+ * );
+ * ```
+ *
+ * Construction throws on invalid config (server boot — where it should).
+ * The middleware calls `next()` IMMEDIATELY and attaches a `finish`
+ * listener, so the real status code is reported and the request is never
+ * delayed. Everything inside is wrapped; failure is silent.
+ *
+ * Structural types only — no dependency on Express, not even for types.
+ */
+
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { evaluateRequest } from "../track.js";
+import { sendReport } from "../report.js";
+import type { AiCrawlConfig, RequestFacts, ResolvedAiCrawlConfig } from "../types.js";
+
+/** Structural subset of `express.Request` / Node's `IncomingMessage`. */
+export interface NodeLikeRequest {
+  method?: string | undefined;
+  /** Path + query as received (`/docs/x?y=1`). */
+  url?: string | undefined;
+  /** Express keeps the pre-router-mount path here; prefer it when present. */
+  originalUrl?: string | undefined;
+  headers: Record<string, string | string[] | undefined>;
+  /** Express convenience; used for URL scheme when present. */
+  protocol?: string | undefined;
+  socket?: { remoteAddress?: string | undefined } | undefined;
+}
+
+/** Structural subset of `express.Response` / Node's `ServerResponse`. */
+export interface NodeLikeResponse {
+  statusCode?: number | undefined;
+  on(event: "finish", listener: () => void): unknown;
+}
+
+export type NodeLikeNext = (err?: unknown) => void;
+
+function headerLookup(req: NodeLikeRequest): (name: string) => string | null {
+  return (name: string): string | null => {
+    try {
+      const value = req.headers[name.toLowerCase()];
+      if (value === undefined) return null;
+      return Array.isArray(value) ? (value[0] ?? null) : value;
+    } catch {
+      return null;
+    }
+  };
+}
+
+/** Build an absolute URL for the request, preferring `publicOrigin`. */
+function requestUrl(cfg: ResolvedAiCrawlConfig, req: NodeLikeRequest): string {
+  const path = req.originalUrl ?? req.url ?? "/";
+  if (path.indexOf("://") !== -1) return path; // already absolute (rare)
+  if (cfg.publicOrigin !== undefined) return cfg.publicOrigin + path;
+  const hostHeader = req.headers["host"];
+  const host = Array.isArray(hostHeader) ? hostHeader[0] : hostHeader;
+  if (host === undefined || host === "") return path;
+  const proto = req.protocol === "http" ? "http" : "https";
+  return `${proto}://${host}${path}`;
+}
+
+/**
+ * Create the middleware. THROWS at construction (server boot) when the
+ * config is invalid; the returned middleware itself never throws.
+ */
+export function createTracetenAICrawlerMiddleware(
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): (req: NodeLikeRequest, res: NodeLikeResponse, next: NodeLikeNext) => void {
+  const cfg = isResolvedConfig(config) ? config : defineAiCrawlConfig(config);
+
+  return (req, res, next) => {
+    try {
+      const facts: RequestFacts = {
+        method: req.method ?? "GET",
+        url: requestUrl(cfg, req),
+        header: headerLookup(req),
+        socketAddr: req.socket?.remoteAddress,
+      };
+
+      // Evaluate up-front so non-crawler traffic attaches no listener at
+      // all; report on `finish` so the REAL status code is captured.
+      const payload = evaluateRequest(cfg, facts);
+      if (payload !== null) {
+        res.on("finish", () => {
+          try {
+            const status = res.statusCode;
+            if (typeof status === "number" && status >= 100 && status <= 599) {
+              payload.status = status;
+            }
+            void sendReport(cfg, payload);
+          } catch {
+            /* silent by contract */
+          }
+        });
+      }
+    } catch {
+      /* silent by contract */
+    }
+    next();
+  };
+}

package/src/adapters/hono.ts

@@ -0,0 +1,89 @@
+/**
+ * Hono adapter + generic `Request`/`Response` helper.
+ *
+ * Hono (structural types — no dependency on hono):
+ * ```ts
+ * import { aiCrawlerTracking } from "@traceten/ai-crawl/hono";
+ * app.use("*", aiCrawlerTracking({ siteId: "…", authToken: "tt_bot_…" }));
+ * ```
+ *
+ * Any framework that exposes fetch-API `Request`/`Response`:
+ * ```ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerFetch } from "@traceten/ai-crawl/hono";
+ * const config = defineAiCrawlConfig({ … });
+ * // after producing the response:
+ * trackAICrawlerFetch(request, response, config, ctx?.waitUntil?.bind(ctx));
+ * ```
+ */
+
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts, type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+
+/**
+ * Generic helper for any `Request`/`Response` runtime. Never throws; pass a
+ * `waitUntil` when the runtime has one, otherwise delivery is
+ * fire-and-forget.
+ */
+export function trackAICrawlerFetch(
+  request: FetchLikeRequest,
+  response: { status: number } | undefined,
+  config: ResolvedAiCrawlConfig,
+  waitUntil?: ((promise: Promise<unknown>) => void) | undefined,
+): void {
+  try {
+    if (!isResolvedConfig(config)) return; // raw config: refuse silently, never throw per-request
+    trackFacts(
+      config,
+      factsFromFetchRequest(request),
+      response !== undefined && typeof response.status === "number" ? response.status : undefined,
+      waitUntil,
+    );
+  } catch {
+    /* silent by contract */
+  }
+}
+
+/** Structural subset of Hono's `Context`. */
+export interface HonoLikeContext {
+  req: { raw: FetchLikeRequest };
+  res: { status: number };
+  /** Throws on runtimes without an execution context — accessed inside try. */
+  executionCtx?: { waitUntil(promise: Promise<unknown>): void };
+}
+
+/**
+ * Hono middleware factory. THROWS at construction when the config is
+ * invalid; the middleware itself never throws its own errors (route errors
+ * propagate untouched).
+ */
+export function aiCrawlerTracking(
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): (c: HonoLikeContext, next: () => Promise<void>) => Promise<void> {
+  const cfg = isResolvedConfig(config) ? config : defineAiCrawlConfig(config);
+
+  return async (c, next) => {
+    // Route errors propagate untouched — tracking must not change behaviour.
+    await next();
+    try {
+      let waitUntil: ((p: Promise<unknown>) => void) | undefined;
+      try {
+        const ctx = c.executionCtx; // getter throws on Node's hono server
+        if (ctx !== undefined && typeof ctx.waitUntil === "function") {
+          waitUntil = (p) => ctx.waitUntil(p);
+        }
+      } catch {
+        waitUntil = undefined;
+      }
+      trackFacts(
+        cfg,
+        factsFromFetchRequest(c.req.raw),
+        typeof c.res?.status === "number" ? c.res.status : undefined,
+        waitUntil,
+      );
+    } catch {
+      /* silent by contract */
+    }
+  };
+}