@sprqvntrs/bot-verify 0.1.1

package/src/ranges.ts

@@ -0,0 +1,142 @@
+/**
+ * Google crawler IP range store.
+ *
+ * Holds the union of all three Google published CIDR lists and supports
+ * background refresh. Never empty: seeds from bundled JSON at construction
+ * and falls back to the last-good list on any refresh failure (fail-open).
+ *
+ * @packageDocumentation
+ */
+
+import googlebotData from './data/googlebot.json';
+import specialCrawlersData from './data/special-crawlers.json';
+import userTriggeredData from './data/user-triggered-fetchers.json';
+import { ipInAnyCidr } from './cidr.js';
+
+/** Official URLs for Google's published crawler IP ranges. */
+export const GOOGLE_RANGE_URLS = [
+  'https://developers.google.com/static/search/apis/ipranges/googlebot.json',
+  'https://developers.google.com/static/search/apis/ipranges/special-crawlers.json',
+  'https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json',
+] as const;
+
+/**
+ * Extracts CIDR strings from a Google IP range JSON payload.
+ * Handles unknown input defensively — never throws.
+ */
+export function parsePrefixes(data: unknown): string[] {
+  if (typeof data !== 'object' || data === null || !('prefixes' in data)) {
+    return [];
+  }
+
+  const prefixes = (data as { prefixes: unknown }).prefixes;
+  if (!Array.isArray(prefixes)) {
+    return [];
+  }
+
+  const result: string[] = [];
+  for (const entry of prefixes) {
+    if (typeof entry !== 'object' || entry === null) {
+      continue;
+    }
+
+    const e = entry as Record<string, unknown>;
+    if (typeof e['ipv4Prefix'] === 'string') {
+      result.push(e['ipv4Prefix']);
+    } else if (typeof e['ipv6Prefix'] === 'string') {
+      result.push(e['ipv6Prefix']);
+    }
+  }
+
+  return result;
+}
+
+/** The bundled (fail-safe) union of all three Google range files. */
+const BUNDLED_CIDRS: readonly string[] = [
+  ...parsePrefixes(googlebotData),
+  ...parsePrefixes(specialCrawlersData),
+  ...parsePrefixes(userTriggeredData),
+];
+
+/** Options for constructing a {@link RangeStore}. */
+export interface RangeStoreOptions {
+  /**
+   * Override the initial CIDR list (useful in tests; pass `[]` to simulate
+   * empty ranges). Defaults to the bundled union.
+   */
+  initialRanges?: string[];
+  /**
+   * Injectable clock for testability. Defaults to `Date.now`.
+   */
+  now?: () => number;
+}
+
+/**
+ * Holds the current union of Google crawler CIDR ranges and provides
+ * background refresh from the official Google endpoints.
+ *
+ * Construction is synchronous and never leaves the store empty (seeds from
+ * bundled data immediately). Refresh is async and fail-open.
+ */
+export class RangeStore {
+  private cidrs: string[];
+  private readonly now: () => number;
+  lastRefreshedAt: number;
+
+  constructor(opts?: RangeStoreOptions) {
+    this.cidrs = opts?.initialRanges !== undefined ? [...opts.initialRanges] : [...BUNDLED_CIDRS];
+    this.now = opts?.now ?? Date.now;
+    this.lastRefreshedAt = this.now();
+  }
+
+  /**
+   * Returns true if `ip` is contained within the current CIDR union.
+   */
+  contains(ip: string): boolean {
+    return ipInAnyCidr(ip, this.cidrs);
+  }
+
+  /**
+   * Fetches all three Google range URLs and atomically replaces the current
+   * CIDR union only if ALL fetches succeed.
+   *
+   * On any failure (network error, parse error, unexpected shape), keeps the
+   * last-good list and returns `false`. Never throws, never empties the store.
+   *
+   * @param fetchImpl - Fetch implementation (injectable for tests)
+   */
+  async refresh(fetchImpl: typeof fetch = fetch): Promise<boolean> {
+    try {
+      const responses = await Promise.all(
+        GOOGLE_RANGE_URLS.map((url) => fetchImpl(url)),
+      );
+
+      // Fail if any response was not OK
+      for (const resp of responses) {
+        if (!resp.ok) {
+          return false;
+        }
+      }
+
+      const bodies = await Promise.all(responses.map((r) => r.json() as Promise<unknown>));
+
+      const newCidrs: string[] = [];
+      for (const body of bodies) {
+        const prefixes = parsePrefixes(body);
+        if (prefixes.length === 0) {
+          // Unexpected shape — bail out to keep last-good
+          return false;
+        }
+        newCidrs.push(...prefixes);
+      }
+
+      // Atomic replace
+      this.cidrs = newCidrs;
+      this.lastRefreshedAt = this.now();
+      return true;
+    } catch {
+      // Network errors, JSON parse errors, etc. — keep last-good list
+      return false;
+    }
+  }
+}

package/src/rdns.ts

@@ -0,0 +1,167 @@
+/**
+ * Reverse-DNS verification for Google crawlers.
+ *
+ * Implements the two-step check recommended by Google:
+ *   1. PTR lookup on the IP → hostname
+ *   2. Forward A/AAAA lookup on that hostname → must contain the original IP
+ *
+ * @packageDocumentation
+ */
+
+import dns from 'node:dns/promises';
+import { normalizeIp } from './cidr.js';
+
+const DNS_ERROR_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ETIMEOUT', 'ESERVFAIL', 'ENONAME']);
+
+/** Allowed domains for reverse-DNS confirmation. */
+const DEFAULT_ALLOWED_DOMAINS = ['googlebot.com', 'google.com', 'googleusercontent.com'] as const;
+
+/** Options for {@link reverseDnsVerify}. */
+export interface ReverseDnsOptions {
+  /**
+   * Hostname suffixes that are accepted as legitimate Google crawler domains.
+   * Defaults to `['googlebot.com', 'google.com', 'googleusercontent.com']`.
+   */
+  allowedDomains?: string[];
+  /**
+   * Maximum time in milliseconds to wait for DNS lookups.
+   * Defaults to `1500`.
+   */
+  timeoutMs?: number;
+}
+
+/**
+ * Performs a two-step reverse-DNS verification for a Google crawler IP.
+ *
+ * Returns:
+ * - `'confirmed'` — PTR resolves to a Google domain AND forward A/AAAA
+ *   resolves back to the original IP
+ * - `'failed'` — DNS resolves successfully but does NOT confirm Google
+ *   (no PTR, non-Google hostname, or forward mismatch)
+ * - `'error'` — DNS infrastructure failure (timeout, SERVFAIL, ECONNREFUSED)
+ *
+ * A timeout is treated as `'error'` (uncertain), not `'failed'` (spoofed),
+ * because we must not block legitimate bots due to DNS unreachability.
+ */
+export async function reverseDnsVerify(
+  ip: string,
+  opts?: ReverseDnsOptions,
+): Promise<'confirmed' | 'failed' | 'error'> {
+  const allowedDomains = opts?.allowedDomains ?? DEFAULT_ALLOWED_DOMAINS;
+  const timeoutMs = opts?.timeoutMs ?? 1500;
+
+  return Promise.race([
+    doVerify(ip, allowedDomains),
+    timeout(timeoutMs),
+  ]);
+}
+
+async function doVerify(
+  ip: string,
+  allowedDomains: readonly string[],
+): Promise<'confirmed' | 'failed' | 'error'> {
+  const normalizedOriginal = normalizeIp(ip);
+  if (normalizedOriginal === null) {
+    return 'failed';
+  }
+
+  // Step 1: PTR lookup
+  let hostnames: string[];
+  try {
+    hostnames = await dns.reverse(ip);
+  } catch (err) {
+    return classifyDnsError(err);
+  }
+
+  if (hostnames.length === 0) {
+    return 'failed';
+  }
+
+  // Step 2: Forward lookup on each hostname that matches an allowed domain
+  for (const hostname of hostnames) {
+    if (!isAllowedDomain(hostname, allowedDomains)) {
+      continue;
+    }
+
+    const forwardConfirmed = await forwardLookup(hostname, normalizedOriginal);
+    if (forwardConfirmed === 'confirmed') {
+      return 'confirmed';
+    }
+    if (forwardConfirmed === 'error') {
+      return 'error';
+    }
+    // 'failed' → try next hostname
+  }
+
+  return 'failed';
+}
+
+async function forwardLookup(
+  hostname: string,
+  originalIp: string,
+): Promise<'confirmed' | 'failed' | 'error'> {
+  try {
+    const [ipv4Results, ipv6Results] = await Promise.allSettled([
+      dns.resolve4(hostname),
+      dns.resolve6(hostname),
+    ]);
+
+    const addresses: string[] = [];
+
+    if (ipv4Results.status === 'fulfilled') {
+      addresses.push(...ipv4Results.value);
+    } else if (isInfraError(ipv4Results.reason)) {
+      return 'error';
+    }
+
+    if (ipv6Results.status === 'fulfilled') {
+      addresses.push(...ipv6Results.value);
+    } else if (isInfraError(ipv6Results.reason)) {
+      return 'error';
+    }
+
+    for (const addr of addresses) {
+      const normalized = normalizeIp(addr);
+      if (normalized !== null && normalized === originalIp) {
+        return 'confirmed';
+      }
+    }
+
+    return 'failed';
+  } catch (err) {
+    return isInfraError(err) ? 'error' : 'failed';
+  }
+}
+
+function isAllowedDomain(hostname: string, allowedDomains: readonly string[]): boolean {
+  const lower = hostname.toLowerCase();
+  for (const domain of allowedDomains) {
+    if (lower === domain || lower.endsWith(`.${domain}`)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function classifyDnsError(err: unknown): 'failed' | 'error' {
+  if (isInfraError(err)) {
+    return 'error';
+  }
+  // ENOTFOUND, ENODATA, NXDOMAIN, ENONAME → legitimately no record → failed
+  return 'failed';
+}
+
+function isInfraError(err: unknown): boolean {
+  if (typeof err !== 'object' || err === null) {
+    return false;
+  }
+  const code = (err as { code?: string }).code;
+  if (code === undefined) {
+    return false;
+  }
+  return DNS_ERROR_CODES.has(code);
+}
+
+function timeout(ms: number): Promise<'error'> {
+  return new Promise((resolve) => setTimeout(() => resolve('error'), ms));
+}

package/src/types.ts

@@ -0,0 +1,45 @@
+/**
+ * Core types for bot verification.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * The outcome of a bot verification check.
+ *
+ * - `'verified'` — IP confirmed as a legitimate Google crawler
+ * - `'spoofed'` — IP does NOT belong to Google; the UA claim is fraudulent
+ * - `'uncertain'` — could not confirm or deny (e.g., DNS error, missing IP)
+ * - `'not-a-claim'` — the User-Agent does not claim to be a Google crawler
+ */
+export type BotVerdict = 'verified' | 'spoofed' | 'uncertain' | 'not-a-claim';
+
+/** Input supplied to the verifier for a single request. */
+export interface VerifyInput {
+  /** Raw User-Agent header value. */
+  userAgent: string;
+  /** Client IP address (may be null if unavailable). */
+  ip: string | null;
+}
+
+/** Result of a single verification. */
+export interface VerifyResult {
+  /** Final verdict for this request. */
+  verdict: BotVerdict;
+  /**
+   * The Google crawler token that was claimed in the UA, e.g. `'Googlebot'`.
+   * `null` when `verdict` is `'not-a-claim'`.
+   */
+  claimedBot: string | null;
+  /** The client IP that was evaluated (may be null when not available). */
+  ip: string | null;
+  /**
+   * The method used to reach the verdict.
+   * - `'ip-range'` — checked against Google's published CIDR lists
+   * - `'rdns'` — verified (or failed) via reverse-DNS lookup
+   * - `'none'` — no IP/network check was performed
+   */
+  method: 'ip-range' | 'rdns' | 'none';
+  /** Short human-readable explanation of the verdict. */
+  reason: string;
+}

package/src/ua.ts

@@ -0,0 +1,75 @@
+/**
+ * User-Agent crawler detection for Google's crawler family.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * All Google crawler tokens whose presence in a UA string constitutes a
+ * verifiable claim. Ordered from most-specific to least-specific so the
+ * detection loop returns the best match.
+ */
+export const GOOGLE_CRAWLER_TOKENS = [
+  'AdsBot-Google-Mobile',
+  'AdsBot-Google',
+  'Googlebot-Image',
+  'Googlebot-News',
+  'Googlebot-Video',
+  'Googlebot-Mobile',
+  'Google-InspectionTool',
+  'Storebot-Google',
+  'Google-Read-Aloud',
+  'Google-Site-Verification',
+  'FeedFetcher-Google',
+  'Mediapartners-Google',
+  'GoogleOther',
+  'APIs-Google',
+  'Googlebot',
+] as const;
+
+export type GoogleCrawlerToken = (typeof GOOGLE_CRAWLER_TOKENS)[number];
+
+/**
+ * Detects whether a User-Agent string claims to be a Google crawler.
+ *
+ * Uses word-boundary matching (not a loose substring) to avoid false positives
+ * from UAs that merely contain the word "google" incidentally (e.g. a normal
+ * Chrome browser whose referrer is embedded in the UA).
+ *
+ * Returns the most specific matched token (e.g. `'Googlebot-Image'` rather
+ * than `'Googlebot'`), or `null` if no Google crawler token is present.
+ *
+ * @param userAgent - The raw User-Agent header value
+ * @returns The matched crawler token or `null`
+ *
+ * @example
+ * ```typescript
+ * detectClaimedCrawler('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')
+ * // => 'Googlebot'
+ *
+ * detectClaimedCrawler('Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 ...')
+ * // => null
+ * ```
+ */
+export function detectClaimedCrawler(userAgent: string): GoogleCrawlerToken | null {
+  if (!userAgent) {
+    return null;
+  }
+
+  for (const token of GOOGLE_CRAWLER_TOKENS) {
+    // Alphanumeric/hyphen lookarounds: the token must not be immediately
+    // preceded or followed by A-Z, a-z, 0-9, or hyphen. This handles
+    // close-parens, dots, slashes, end-of-string, etc. as valid boundaries
+    // while still preventing embedded matches inside longer identifiers.
+    const pattern = new RegExp(`(?<![A-Za-z0-9-])${escapeRegExp(token)}(?![A-Za-z0-9-])`, 'i');
+    if (pattern.test(userAgent)) {
+      return token;
+    }
+  }
+
+  return null;
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

package/src/verify.ts

@@ -0,0 +1,209 @@
+/**
+ * Core bot verification logic.
+ *
+ * Creates a `BotVerifier` that checks whether a request claiming to be a
+ * Google crawler actually originates from Google's published IP ranges or can
+ * be confirmed via reverse-DNS.
+ *
+ * OVERRIDING CORRECTNESS RULE: NEVER classify a real Google crawler as
+ * `'spoofed'`. When in doubt (missing IP, DNS errors, range refresh failure),
+ * return `'uncertain'` so callers treat the request as pass-through.
+ *
+ * @packageDocumentation
+ */
+
+import { detectClaimedCrawler } from './ua.js';
+import { normalizeIp } from './cidr.js';
+import { RangeStore } from './ranges.js';
+import { reverseDnsVerify } from './rdns.js';
+import type { VerifyInput, VerifyResult } from './types.js';
+
+/** Options for {@link createBotVerifier}. */
+export interface BotVerifierOptions {
+  /**
+   * How often to refresh the Google IP ranges from the official URLs.
+   * Defaults to 24 hours.
+   */
+  rangeRefreshTtlMs?: number;
+  /**
+   * Whether to perform a reverse-DNS lookup when the IP is not in the
+   * published ranges. Defaults to `true`.
+   *
+   * When `false`, any IP not in the published ranges is immediately classified
+   * as `'spoofed'`.
+   */
+  rdns?: boolean;
+  /**
+   * Timeout for rDNS lookups in milliseconds. Defaults to `1500`.
+   */
+  rdnsTimeoutMs?: number;
+  /**
+   * Injectable fetch implementation (useful in tests to avoid network calls).
+   * Defaults to the global `fetch`.
+   */
+  fetchImpl?: typeof fetch;
+  /**
+   * Injectable rDNS implementation (useful in tests).
+   * Defaults to {@link reverseDnsVerify} with `rdnsTimeoutMs` applied.
+   */
+  rdnsImpl?: (ip: string) => Promise<'confirmed' | 'failed' | 'error'>;
+  /**
+   * Injectable clock for testability. Defaults to `Date.now`.
+   */
+  now?: () => number;
+  /**
+   * Override the initial CIDR list (useful in tests; pass `[]` to simulate
+   * empty ranges). Defaults to the bundled union of all three Google files.
+   */
+  initialRanges?: string[];
+  /**
+   * Optional structured logger called after every verification.
+   * Errors thrown by the logger are swallowed so they never affect the result.
+   */
+  logger?: (result: VerifyResult) => void;
+}
+
+/** Public interface for a bot verifier instance. */
+export interface BotVerifier {
+  /**
+   * Verifies whether the request described by `input` is a legitimate
+   * Google crawler or a spoof attempt.
+   */
+  verify(input: VerifyInput): Promise<VerifyResult>;
+  /**
+   * Manually triggers a refresh of the Google IP ranges.
+   * Returns `true` on success, `false` if the refresh failed (ranges unchanged).
+   */
+  refreshRanges(): Promise<boolean>;
+}
+
+/**
+ * Creates a new {@link BotVerifier} instance.
+ *
+ * The verifier seeds its IP range list from the bundled JSON files
+ * immediately (so it is never empty) and refreshes lazily in the background
+ * based on `rangeRefreshTtlMs`.
+ */
+export function createBotVerifier(opts?: BotVerifierOptions): BotVerifier {
+  const rangeRefreshTtlMs = opts?.rangeRefreshTtlMs ?? 24 * 60 * 60 * 1000;
+  const useRdns = opts?.rdns ?? true;
+  const rdnsTimeoutMs = opts?.rdnsTimeoutMs ?? 1500;
+  const fetchImpl = opts?.fetchImpl ?? fetch;
+  const nowFn = opts?.now ?? Date.now;
+  const logger = opts?.logger;
+
+  const store = new RangeStore({
+    initialRanges: opts?.initialRanges,
+    now: nowFn,
+  });
+
+  const rdnsImpl =
+    opts?.rdnsImpl ??
+    ((ip: string) => reverseDnsVerify(ip, { timeoutMs: rdnsTimeoutMs }));
+
+  async function verify(input: VerifyInput): Promise<VerifyResult> {
+    const result = await doVerify(input);
+    if (logger) {
+      try {
+        logger(result);
+      } catch {
+        // Logger errors must never propagate
+      }
+    }
+    return result;
+  }
+
+  async function doVerify(input: VerifyInput): Promise<VerifyResult> {
+    const { userAgent, ip } = input;
+
+    // Step 1: Check if the UA claims to be a Google crawler at all
+    const claimedBot = detectClaimedCrawler(userAgent);
+    if (claimedBot === null) {
+      return {
+        verdict: 'not-a-claim',
+        claimedBot: null,
+        ip,
+        method: 'none',
+        reason: 'UA does not claim a Google crawler',
+      };
+    }
+
+    // Step 2: Validate the IP
+    const normalizedIp = ip ? normalizeIp(ip) : null;
+    if (!ip || normalizedIp === null) {
+      return {
+        verdict: 'uncertain',
+        claimedBot,
+        ip: ip ?? null,
+        method: 'none',
+        reason: 'no usable client IP',
+      };
+    }
+
+    // Step 3: Lazy range refresh
+    const age = nowFn() - store.lastRefreshedAt;
+    if (age > rangeRefreshTtlMs) {
+      // Fire and forget — failures are swallowed inside RangeStore.refresh
+      await store.refresh(fetchImpl);
+    }
+
+    // Step 4: IP-range check
+    if (store.contains(normalizedIp)) {
+      return {
+        verdict: 'verified',
+        claimedBot,
+        ip: normalizedIp,
+        method: 'ip-range',
+        reason: 'IP in Google published ranges',
+      };
+    }
+
+    // Step 5: rDNS check (when enabled)
+    if (useRdns) {
+      const rdnsResult = await rdnsImpl(normalizedIp);
+
+      if (rdnsResult === 'confirmed') {
+        return {
+          verdict: 'verified',
+          claimedBot,
+          ip: normalizedIp,
+          method: 'rdns',
+          reason: 'reverse-DNS confirmed Google',
+        };
+      }
+
+      if (rdnsResult === 'failed') {
+        return {
+          verdict: 'spoofed',
+          claimedBot,
+          ip: normalizedIp,
+          method: 'rdns',
+          reason: 'reverse-DNS does not confirm Google ownership',
+        };
+      }
+
+      // rdnsResult === 'error'
+      return {
+        verdict: 'uncertain',
+        claimedBot,
+        ip: normalizedIp,
+        method: 'rdns',
+        reason: 'rDNS inconclusive (DNS infrastructure error)',
+      };
+    }
+
+    // Step 6: rDNS disabled — IP not in range → spoofed
+    return {
+      verdict: 'spoofed',
+      claimedBot,
+      ip: normalizedIp,
+      method: 'ip-range',
+      reason: 'IP not in Google published ranges (rDNS disabled)',
+    };
+  }
+
+  return {
+    verify,
+    refreshRanges: () => store.refresh(fetchImpl),
+  };
+}