@traceten/ai-crawl 0.1.0

package/src/index.ts

@@ -0,0 +1,46 @@
+/**
+ * @traceten/ai-crawl — server-side AI crawler tracking.
+ *
+ * AI crawlers (GPTBot, ClaudeBot, PerplexityBot, …) never execute
+ * JavaScript, so the browser snippet cannot see them. The observation point
+ * has to be the customer's server — this package is that observation point.
+ * It pre-filters locally, matches user agents with anchored tokens, and
+ * reports plausible AI crawls to `POST /v1/ai-crawls`. Classification,
+ * verification and confidence are decided server-side.
+ *
+ * Adapters: `@traceten/ai-crawl/next`, `/cloudflare-pages`,
+ * `/cloudflare-workers`, `/express`, `/hono`.
+ */
+
+export {
+  AUTH_TOKEN_PREFIX,
+  DEFAULT_ALLOWED_METHODS,
+  DEFAULT_ENDPOINT,
+  defineAiCrawlConfig,
+  isResolvedConfig,
+} from "./config.js";
+export {
+  DEFAULT_DENY_EXTENSIONS,
+  DEFAULT_DENY_PATH_PREFIXES,
+  DENY_SEC_FETCH_DESTS,
+  isCrawlerFacingPath,
+  passesPreFilter,
+} from "./filter.js";
+export { matchCrawler } from "./matcher.js";
+export { resolveCrawlerIp } from "./ip.js";
+export { REPORT_TIMEOUT_MS, buildPayload, sendReport, type SendOptions } from "./report.js";
+export {
+  evaluateRequest,
+  factsFromFetchRequest,
+  trackFacts,
+  type FetchLikeRequest,
+} from "./track.js";
+export type {
+  AiCrawlConfig,
+  AiCrawlDeliveryError,
+  AiCrawlWirePayload,
+  CrawlerCategory,
+  CrawlerMatch,
+  RequestFacts,
+  ResolvedAiCrawlConfig,
+} from "./types.js";

package/src/ip.ts

@@ -0,0 +1,112 @@
+/**
+ * Crawler IP resolution.
+ *
+ * The crawler's TCP connection terminates at the CUSTOMER's server, so the
+ * crawler IP is only observable here — Traceten's edge sees the customer's
+ * origin IP, not the crawler's. This module
+ * derives the best-available value; the payload OMITS the field rather than
+ * send a wrong one.
+ *
+ * Resolution order:
+ *   1. `cf-connecting-ip` — ONLY when `trustCfConnectingIp` is set. The
+ *      Cloudflare adapters set it automatically (the platform strips and
+ *      rewrites the header there); anywhere else it is client-forgeable,
+ *      and a forged vendor-range IP paired with a vendor UA is exactly the
+ *      spoof the server-side verification exists to catch.
+ *   2. `x-forwarded-for` — ONLY when `trustProxy` is true, selecting the
+ *      entry `proxyDepth` hops from the right (each trusted proxy appends
+ *      one entry; anything further left is attacker-controllable)
+ *   3. Socket remote address, where the runtime exposes one
+ */
+
+import type { RequestFacts, ResolvedAiCrawlConfig } from "./types.js";
+
+const IPV4_RE = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
+// Loose IPv6 shape check — up to 8 hex groups separated by colons ("::"
+// compression appears as empty groups). Deliberately permissive: this is a
+// plausibility gate, not a validator; the edge re-validates.
+const IPV6_RE = /^[0-9a-f]{0,4}(:[0-9a-f]{0,4}){1,7}$/i;
+
+/** Normalise a candidate; returns `undefined` when it is not a plausible IP. */
+export function normalizeIp(raw: string | null | undefined): string | undefined {
+  if (!raw) return undefined;
+  let value = raw.trim();
+  if (value === "") return undefined;
+
+  // Bracketed IPv6, possibly with a port: [::1]:8080
+  if (value.startsWith("[")) {
+    const close = value.indexOf("]");
+    if (close === -1) return undefined;
+    value = value.slice(1, close);
+  } else {
+    // IPv4 with a port: 1.2.3.4:5678 (a bare IPv6 also contains ":", so only
+    // strip when what remains is a valid IPv4).
+    const colon = value.indexOf(":");
+    if (colon !== -1 && value.indexOf(":", colon + 1) === -1) {
+      const head = value.slice(0, colon);
+      if (IPV4_RE.test(head)) value = head;
+    }
+  }
+
+  // Node reports IPv4-mapped addresses as ::ffff:1.2.3.4.
+  if (value.toLowerCase().startsWith("::ffff:")) {
+    const tail = value.slice(7);
+    if (IPV4_RE.test(tail)) value = tail;
+  }
+
+  const v4 = IPV4_RE.exec(value);
+  if (v4 !== null) {
+    for (let i = 1; i <= 4; i++) {
+      if (Number(v4[i]) > 255) return undefined;
+    }
+    return value;
+  }
+  if (value.includes(":") && value.length <= 45 && IPV6_RE.test(value)) {
+    return value.toLowerCase();
+  }
+  return undefined;
+}
+
+/**
+ * Select the client entry from an `x-forwarded-for` value given the number
+ * of trusted proxies that append to it. Returns `undefined` when the chain
+ * is shorter than `proxyDepth` (a wrong value is worse than none).
+ */
+export function ipFromForwardedFor(headerValue: string, proxyDepth: number): string | undefined {
+  const entries = headerValue
+    .split(",")
+    .map((e) => e.trim())
+    .filter((e) => e !== "");
+  if (entries.length === 0 || proxyDepth < 1 || entries.length < proxyDepth) {
+    return undefined;
+  }
+  return normalizeIp(entries[entries.length - proxyDepth]);
+}
+
+/**
+ * Resolve the crawler IP from a request. Returns `undefined` when no
+ * trustworthy value is derivable. Never throws.
+ */
+export function resolveCrawlerIp(
+  cfg: ResolvedAiCrawlConfig,
+  facts: RequestFacts,
+): string | undefined {
+  try {
+    if (cfg.trustCfConnectingIp) {
+      const cf = normalizeIp(facts.header("cf-connecting-ip"));
+      if (cf !== undefined) return cf;
+    }
+
+    if (cfg.trustProxy) {
+      const xff = facts.header("x-forwarded-for");
+      if (xff) {
+        const fromXff = ipFromForwardedFor(xff, cfg.proxyDepth);
+        if (fromXff !== undefined) return fromXff;
+      }
+    }
+
+    return normalizeIp(facts.socketAddr);
+  } catch {
+    return undefined;
+  }
+}

package/src/matcher.ts

@@ -0,0 +1,119 @@
+/**
+ * Anchored two-tier user-agent matching.
+ *
+ * NEVER `includes()` on the whole string — that is the reference package's
+ * defect (its page claims token matching; its code does substring matching,
+ * so a browser UA containing `grok` or `copilot` false-positives). Follows
+ * the same anchoring discipline Traceten's server-side classifier uses,
+ * adapted for crawler UAs where the token sits mid-string
+ * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).
+ *
+ * Boundary rule: a token matches only when the character before it and (for
+ * tier 1) the character after it are NOT token characters `[a-z0-9._-]`.
+ * So `compatible; GPTBot/1.1` matches, and `not-really-GPTBot` does not
+ * (preceded by `-`), and `GPTBotnet` does not (followed by `n`).
+ */
+
+import { AGENT_TOKENS, PROVIDER_ALIASES } from "./crawlers.js";
+import type { CrawlerMatch } from "./types.js";
+
+/** Characters that can appear inside a UA product token. */
+function isTokenChar(ch: string): boolean {
+  return (
+    (ch >= "a" && ch <= "z") || (ch >= "0" && ch <= "9") || ch === "." || ch === "_" || ch === "-"
+  );
+}
+
+/** Anchoring mode. */
+export type AnchorMode = "token" | "prefix";
+
+/**
+ * True when `needle` occurs in `haystack` (both lowercase) anchored at a
+ * token boundary.
+ *
+ * `"token"` (tier 1): boundary required on BOTH sides — `GPTBot/1.1`
+ * matches, `not-really-GPTBot` and `GPTBotnet` do not.
+ *
+ * `"prefix"` (tier 2): left boundary required; the right side must be a
+ * boundary OR a compound-token delimiter (`-`, `_`, `.`) — so `claude`
+ * catches a future `Claude-NewAgent/1.0` without matching `claudius`.
+ * A needle that itself ends in a delimiter (e.g. `oai-`, `meta-`) is a free
+ * prefix past that delimiter.
+ */
+export function hasAnchoredToken(haystack: string, needle: string, mode: AnchorMode): boolean {
+  const needleEndsWithDelimiter =
+    needle.endsWith("-") || needle.endsWith("_") || needle.endsWith(".");
+  let idx = haystack.indexOf(needle);
+  while (idx !== -1) {
+    let leftOk = idx === 0 || !isTokenChar(haystack.charAt(idx - 1));
+    // Prefix (tier-2) matches skip occurrences that start a URL host —
+    // `SomeScraper/1.0 (+https://anthropic.com/policies)` is a scraper
+    // citing a vendor page, not a vendor agent. Tier-1 tokens keep matching
+    // anywhere: real crawler tokens never begin a hostname.
+    if (leftOk && mode === "prefix" && idx >= 3 && haystack.slice(idx - 3, idx) === "://") {
+      leftOk = false;
+    }
+    const rightIdx = idx + needle.length;
+    const nextCh = rightIdx >= haystack.length ? "" : haystack.charAt(rightIdx);
+    let rightOk: boolean;
+    if (mode === "token") {
+      rightOk = nextCh === "" || !isTokenChar(nextCh);
+    } else {
+      rightOk =
+        needleEndsWithDelimiter ||
+        nextCh === "" ||
+        !isTokenChar(nextCh) ||
+        nextCh === "-" ||
+        nextCh === "_" ||
+        nextCh === ".";
+    }
+    if (leftOk && rightOk) return true;
+    idx = haystack.indexOf(needle, idx + 1);
+  }
+  return false;
+}
+
+/**
+ * Match a raw User-Agent string against the local crawler list.
+ *
+ * Tier 1: exact agent tokens, boundary-anchored on both sides.
+ * Tier 2: coarse provider aliases, left-boundary-anchored prefixes, so a new
+ * agent from a known vendor is captured without a release. Tier-2 matches
+ * carry category `ai_crawler` locally; the server assigns the real one.
+ *
+ * Returns `null` for no match. Never throws.
+ */
+export function matchCrawler(userAgent: string | null | undefined): CrawlerMatch | null {
+  if (!userAgent) return null;
+  let ua: string;
+  try {
+    ua = userAgent.toLowerCase();
+  } catch {
+    return null;
+  }
+  if (ua.length === 0 || ua.length > 4096) return null;
+
+  for (const entry of AGENT_TOKENS) {
+    if (hasAnchoredToken(ua, entry.token, "token")) {
+      return {
+        agent: entry.agent,
+        provider: entry.provider,
+        category: entry.category,
+        tier: "exact",
+      };
+    }
+  }
+
+  for (const entry of PROVIDER_ALIASES) {
+    if (hasAnchoredToken(ua, entry.alias, "prefix")) {
+      return {
+        agent: null,
+        provider: entry.provider,
+        category: "ai_crawler",
+        tier: "provider",
+      };
+    }
+  }
+
+  return null;
+}

package/src/report.ts

@@ -0,0 +1,149 @@
+/**
+ * Payload construction + delivery.
+ *
+ * Contract: never throws, never blocks the response, 1500 ms
+ * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
+ * silent — a broken analytics call must never surface on a customer's site.
+ *
+ * Silent is not the same as invisible: an opt-in `onError` reports a non-2xx
+ * or a network failure to the caller. Without it a rejected report cannot be
+ * told from a delivered one, and a bad token reads as "no crawlers visited".
+ */
+
+import type {
+  AiCrawlDeliveryError,
+  AiCrawlWirePayload,
+  ResolvedAiCrawlConfig,
+} from "./types.js";
+
+export const REPORT_TIMEOUT_MS = 1500;
+
+/** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
+export function buildReportUrl(cfg: ResolvedAiCrawlConfig, url: string): string {
+  try {
+    if (cfg.publicOrigin === undefined) return url;
+    const schemeIdx = url.indexOf("://");
+    if (schemeIdx === -1) {
+      // Bare path — just prefix the origin.
+      return cfg.publicOrigin + (url.startsWith("/") ? url : `/${url}`);
+    }
+    const pathStart = url.indexOf("/", schemeIdx + 3);
+    return pathStart === -1 ? cfg.publicOrigin + "/" : cfg.publicOrigin + url.slice(pathStart);
+  } catch {
+    return url;
+  }
+}
+
+export interface ReportInput {
+  url: string;
+  method: string;
+  userAgent: string;
+  status?: number | undefined;
+  ip?: string | undefined;
+}
+
+/** Build the wire payload. Optional fields are OMITTED, never null/wrong. */
+export function buildPayload(cfg: ResolvedAiCrawlConfig, input: ReportInput): AiCrawlWirePayload {
+  const payload: AiCrawlWirePayload = {
+    site_id: cfg.siteId,
+    url: buildReportUrl(cfg, input.url),
+    method: input.method.toUpperCase(),
+    user_agent: input.userAgent.slice(0, 1024),
+    ts: Date.now(),
+  };
+  if (typeof input.status === "number" && input.status >= 100 && input.status <= 599) {
+    payload.status = input.status;
+  }
+  if (input.ip !== undefined) {
+    payload.ip = input.ip;
+  }
+  return payload;
+}
+
+/** Abort signal with a 1500 ms timeout, where the runtime supports it. */
+function timeoutSignal(): AbortSignal | undefined {
+  try {
+    if (typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function") {
+      return AbortSignal.timeout(REPORT_TIMEOUT_MS);
+    }
+  } catch {
+    /* ignore */
+  }
+  return undefined;
+}
+
+/** Delivery options. */
+export interface SendOptions {
+  /**
+   * Set `keepalive: true` on the fetch. Default `true` for fire-and-forget
+   * paths (Express, bare fetch) where nothing else keeps delivery alive
+   * past the response. Callers that schedule via `waitUntil` pass `false`:
+   * `waitUntil` already extends the lifetime, and `keepalive` is not
+   * reliably supported on workerd — if the option ever threw there, every
+   * report from the Cloudflare adapters would be lost, silently.
+   */
+  keepalive?: boolean;
+}
+
+/**
+ * Send a report. Returns a promise that ALWAYS resolves — network errors,
+ * timeouts and non-2xx responses are swallowed. Suitable for `waitUntil`
+ * (which needs the promise) and for fire-and-forget (ignore it).
+ */
+export function sendReport(
+  cfg: ResolvedAiCrawlConfig,
+  payload: AiCrawlWirePayload,
+  options?: SendOptions,
+): Promise<void> {
+  try {
+    const fetchImpl = cfg.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined);
+    if (fetchImpl === undefined) return Promise.resolve();
+
+    const signal = timeoutSignal();
+    const init: RequestInit = {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: `Bearer ${cfg.authToken}`,
+      },
+      body: JSON.stringify(payload),
+    };
+    // Omit the property entirely when disabled — safer than `keepalive:
+    // false` on runtimes that reject the option.
+    if (options?.keepalive !== false) init.keepalive = true;
+    if (signal !== undefined) init.signal = signal;
+
+    return fetchImpl(cfg.endpoint, init).then(
+      (res) => {
+        // A rejected report used to be indistinguishable from a delivered one:
+        // both branches discarded their argument, so a 401 from a bad token or
+        // a 403 from blocked egress read as success and the customer saw
+        // "no crawlers" forever. Still silent by default: `onError` is opt-in.
+        if (res !== undefined && res !== null && res.ok !== true) {
+          notifyError(cfg, { kind: "http", status: res.status, endpoint: cfg.endpoint });
+        }
+        return undefined;
+      },
+      (cause: unknown) => {
+        notifyError(cfg, { kind: "network", cause, endpoint: cfg.endpoint });
+        return undefined;
+      },
+    );
+  } catch {
+    return Promise.resolve();
+  }
+}
+
+/**
+ * Hand a delivery failure to the customer's callback.
+ *
+ * Guarded: a throwing `onError` must not escape into the host's response path,
+ * which is the whole reason delivery is silent in the first place.
+ */
+function notifyError(cfg: ResolvedAiCrawlConfig, error: AiCrawlDeliveryError): void {
+  try {
+    cfg.onError?.(error);
+  } catch {
+    // Deliberately empty — see above.
+  }
+}

package/src/track.ts

@@ -0,0 +1,117 @@
+/**
+ * Shared orchestrator used by every adapter:
+ * pre-filter → anchored UA match → category opt-outs → IP resolution → send.
+ *
+ * `trackFacts` never throws and never blocks: it either schedules the send
+ * via the runtime's `waitUntil` or fires-and-forgets it.
+ */
+
+import { passesPreFilter } from "./filter.js";
+import { resolveCrawlerIp } from "./ip.js";
+import { matchCrawler } from "./matcher.js";
+import { buildPayload, sendReport } from "./report.js";
+import type {
+  AiCrawlWirePayload,
+  CrawlerCategory,
+  RequestFacts,
+  ResolvedAiCrawlConfig,
+} from "./types.js";
+
+/** Minimal structural view of a fetch-API `Request` — avoids DOM/runtime type deps. */
+export interface FetchLikeRequest {
+  readonly method: string;
+  readonly url: string;
+  readonly headers: { get(name: string): string | null };
+}
+
+/** Build {@link RequestFacts} from a fetch-API `Request`. Never throws. */
+export function factsFromFetchRequest(request: FetchLikeRequest): RequestFacts {
+  return {
+    method: request.method,
+    url: request.url,
+    header: (name: string): string | null => {
+      try {
+        return request.headers.get(name);
+      } catch {
+        return null;
+      }
+    },
+  };
+}
+
+/** Is this category disabled by config? */
+function categoryDisabled(cfg: ResolvedAiCrawlConfig, category: CrawlerCategory): boolean {
+  switch (category) {
+    case "answer_fetch":
+      return cfg.disableAnswerFetch;
+    case "search_index":
+      return cfg.disableSearchCrawlers;
+    case "training":
+      return cfg.disableTrainingCrawlers;
+    case "ai_crawler":
+      return cfg.disableOtherCrawlers;
+  }
+}
+
+/**
+ * Evaluate a request. Returns the wire payload when it should be reported,
+ * `null` otherwise. Pure apart from `Date.now()`; never throws.
+ */
+export function evaluateRequest(
+  cfg: ResolvedAiCrawlConfig,
+  facts: RequestFacts,
+  status?: number | undefined,
+): AiCrawlWirePayload | null {
+  try {
+    const secFetchDest = facts.header("sec-fetch-dest");
+    if (!passesPreFilter(cfg, facts.method, facts.url, secFetchDest)) return null;
+
+    const userAgent = facts.header("user-agent");
+    const match = matchCrawler(userAgent);
+    if (match === null) return null;
+    if (categoryDisabled(cfg, match.category)) return null;
+
+    return buildPayload(cfg, {
+      url: facts.url,
+      method: facts.method,
+      userAgent: userAgent as string, // non-null: matchCrawler(null) === null
+      status,
+      ip: resolveCrawlerIp(cfg, facts),
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Evaluate and deliver. Uses `waitUntil` where the runtime provides one so
+ * the report outlives the response without ever delaying it; otherwise
+ * fire-and-forget. Never throws.
+ */
+export function trackFacts(
+  cfg: ResolvedAiCrawlConfig,
+  facts: RequestFacts,
+  status?: number | undefined,
+  waitUntil?: ((promise: Promise<unknown>) => void) | undefined,
+): void {
+  try {
+    const payload = evaluateRequest(cfg, facts, status);
+    if (payload === null) return;
+
+    // With a waitUntil, delivery lifetime is already guaranteed — skip
+    // keepalive (unverified on workerd). Without one, keepalive is what
+    // keeps the request alive past the response.
+    const delivery = sendReport(cfg, payload, {
+      keepalive: typeof waitUntil !== "function",
+    });
+    if (typeof waitUntil === "function") {
+      try {
+        waitUntil(delivery);
+      } catch {
+        /* runtime rejected the promise registration — delivery still runs */
+      }
+    }
+  } catch {
+    /* silent by contract */
+  }
+}