@traceten/ai-crawl 0.1.0

package/src/adapters/next.ts

@@ -0,0 +1,87 @@
+/**
+ * Next.js adapter — call from `proxy.ts` (Next ≥15.5; formerly
+ * `middleware.ts`).
+ *
+ * ```ts
+ * // proxy.ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerRequest } from "@traceten/ai-crawl/next";
+ *
+ * const config = defineAiCrawlConfig({
+ *   siteId: process.env.TRACETEN_SITE_ID!,
+ *   authToken: process.env.TRACETEN_CRAWL_TOKEN!,
+ * });
+ *
+ * export function proxy(request: NextRequest, event: NextFetchEvent) {
+ *   trackAICrawlerRequest(request, event, config);
+ *   return NextResponse.next();
+ * }
+ * ```
+ *
+ * Validate the config at module scope with `defineAiCrawlConfig` so a
+ * missing token fails the deploy, not your visitors' requests. If a raw
+ * (unvalidated) config reaches this function it is validated lazily; an
+ * invalid one logs ONE loud error and no-ops — per-request code paths never
+ * throw.
+ *
+ * The middleware runs before the response exists, so no `status` is sent —
+ * the edge treats a crawl report without a status as "served".
+ */
+
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts, type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+
+/** Structural subset of Next's `NextFetchEvent`. */
+export interface WaitUntilEvent {
+  waitUntil(promise: Promise<unknown>): void;
+}
+
+const resolvedCache = new WeakMap<AiCrawlConfig, ResolvedAiCrawlConfig | null>();
+
+/** Resolve lazily, warning loudly ONCE per config object on invalid input. */
+function resolveLazily(
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): ResolvedAiCrawlConfig | null {
+  if (isResolvedConfig(config)) return config;
+  if (resolvedCache.has(config)) return resolvedCache.get(config) ?? null;
+  let resolved: ResolvedAiCrawlConfig | null = null;
+  try {
+    resolved = defineAiCrawlConfig(config);
+  } catch (err) {
+    try {
+      // eslint-disable-next-line no-console
+      console.error(
+        "[@traceten/ai-crawl] invalid config — crawler tracking is DISABLED:",
+        err instanceof Error ? err.message : err,
+      );
+    } catch {
+      /* ignore */
+    }
+  }
+  resolvedCache.set(config, resolved);
+  return resolved;
+}
+
+/**
+ * Report the request if it looks like an AI crawler. Never throws, never
+ * blocks — delivery is scheduled via `event.waitUntil` when available.
+ */
+export function trackAICrawlerRequest(
+  request: FetchLikeRequest,
+  event: WaitUntilEvent | undefined,
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): void {
+  try {
+    const cfg = resolveLazily(config);
+    if (cfg === null) return;
+    trackFacts(
+      cfg,
+      factsFromFetchRequest(request),
+      undefined,
+      event !== undefined ? (p) => event.waitUntil(p) : undefined,
+    );
+  } catch {
+    /* silent by contract */
+  }
+}

package/src/config.ts

@@ -0,0 +1,127 @@
+/**
+ * Config validation + defaulting.
+ *
+ * This is the ONE place the package is allowed to throw: at construction,
+ * in the customer's module scope / server boot — never per-request. A
+ * missing `authToken` must fail loudly here rather than silently sending
+ * requests that the endpoint will 401.
+ */
+
+import { DEFAULT_DENY_EXTENSIONS, DEFAULT_DENY_PATH_PREFIXES } from "./filter.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "./types.js";
+
+export const DEFAULT_ENDPOINT = "https://ingest.traceten.com/v1/ai-crawls";
+export const DEFAULT_ALLOWED_METHODS: readonly string[] = ["GET", "HEAD"];
+export const AUTH_TOKEN_PREFIX = "tt_bot_";
+
+function fail(message: string): never {
+  throw new Error(`[@traceten/ai-crawl] ${message}`);
+}
+
+/**
+ * Validate a config and apply defaults. THROWS on invalid input — call it at
+ * module scope (or in your middleware factory), so a misconfiguration fails
+ * your deploy, not your visitors' requests.
+ */
+export function defineAiCrawlConfig(config: AiCrawlConfig): ResolvedAiCrawlConfig {
+  if (typeof config !== "object" || config === null) {
+    fail("config must be an object");
+  }
+
+  if (typeof config.siteId !== "string" || config.siteId.trim() === "") {
+    fail(
+      "siteId is required — use your site key (the ttid_… data-site value on the install page)",
+    );
+  }
+
+  if (typeof config.authToken !== "string" || config.authToken.trim() === "") {
+    fail(
+      "authToken is required. POST /v1/ai-crawls rejects unauthenticated reports, " +
+        "so without it every report would silently 401. Create a crawl token " +
+        "(tt_bot_...) in the Traceten dashboard and keep it server-side.",
+    );
+  }
+  if (!config.authToken.startsWith(AUTH_TOKEN_PREFIX)) {
+    fail(
+      `authToken must start with "${AUTH_TOKEN_PREFIX}". It looks like you passed ` +
+        "something else (perhaps the public snippet key?) — the crawl token is a " +
+        "server-side secret created in the dashboard.",
+    );
+  }
+
+  const proxyDepth = config.proxyDepth ?? 1;
+  if (!Number.isInteger(proxyDepth) || proxyDepth < 1) {
+    fail(
+      "proxyDepth must be a positive integer (number of trusted proxies appending to x-forwarded-for)",
+    );
+  }
+
+  if (config.endpoint !== undefined) {
+    // https only: the request carries the tt_bot_ Bearer token and crawler
+    // IPs. A typo'd http:// endpoint would leak both in cleartext, silently.
+    // Plain http is allowed solely for local development loopback.
+    const isHttps = /^https:\/\//.test(config.endpoint);
+    const isLocalHttp = /^http:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d+)?(\/|$)/.test(
+      config.endpoint,
+    );
+    if (!isHttps && !isLocalHttp) {
+      fail(
+        "endpoint must be an https:// URL (http:// is allowed only for localhost). " +
+          "Reports carry your tt_bot_ token and crawler IPs — never send them in cleartext.",
+      );
+    }
+  }
+  if (config.publicOrigin !== undefined && !/^https?:\/\/[^/]+$/.test(config.publicOrigin)) {
+    fail('publicOrigin must be an origin with no path, e.g. "https://example.com"');
+  }
+
+  const allowedMethods = (config.allowedMethods ?? DEFAULT_ALLOWED_METHODS).map((m) =>
+    String(m).toUpperCase(),
+  );
+  if (allowedMethods.length === 0) {
+    fail("allowedMethods must not be empty");
+  }
+
+  // Extras EXTEND the built-in lists; they never replace them.
+  const denyPathPrefixes = [
+    ...DEFAULT_DENY_PATH_PREFIXES,
+    // Trailing slashes are stripped so the segment-boundary match in
+    // filter.ts (`=== prefix || startsWith(prefix + "/")`) works either way.
+    ...(config.extraDenyPathPrefixes ?? []).map((p) => String(p).toLowerCase().replace(/\/+$/, "")),
+  ];
+  const denyExtensions = [
+    ...DEFAULT_DENY_EXTENSIONS,
+    ...(config.extraDenyExtensions ?? []).map((e) => {
+      const ext = String(e).toLowerCase();
+      return ext.startsWith(".") ? ext : `.${ext}`;
+    }),
+  ];
+
+  const resolved: ResolvedAiCrawlConfig = {
+    siteId: config.siteId.trim(),
+    authToken: config.authToken.trim(),
+    endpoint: config.endpoint ?? DEFAULT_ENDPOINT,
+    allowedMethods,
+    denyPathPrefixes,
+    denyExtensions,
+    disableAnswerFetch: config.disableAnswerFetch === true,
+    disableSearchCrawlers: config.disableSearchCrawlers === true,
+    disableTrainingCrawlers: config.disableTrainingCrawlers === true,
+    disableOtherCrawlers: config.disableOtherCrawlers === true,
+    trustProxy: config.trustProxy === true,
+    trustCfConnectingIp: config.trustCfConnectingIp === true,
+    proxyDepth,
+    publicOrigin: config.publicOrigin,
+    onError: typeof config.onError === "function" ? config.onError : undefined,
+    fetch: config.fetch,
+    __resolved: true,
+  };
+  return resolved;
+}
+
+/** True when the object already passed through {@link defineAiCrawlConfig}. */
+export function isResolvedConfig(
+  config: AiCrawlConfig | ResolvedAiCrawlConfig,
+): config is ResolvedAiCrawlConfig {
+  return (config as ResolvedAiCrawlConfig).__resolved === true;
+}

package/src/crawlers.ts

@@ -0,0 +1,269 @@
+/**
+ * Local crawler token data — a deliberately-loose COST FILTER, not the source
+ * of truth. Traceten's server-side crawler registry decides provider,
+ * category, verification and confidence; this list only decides "worth
+ * reporting". Keep it small and keep every token anchored against current
+ * vendor documentation.
+ *
+ * Recent reconciliation notes, kept because they explain non-obvious entries:
+ *   - Deleted `GoogleAgent` (a hyphenation typo of the real `Google-Agent`,
+ *     which stays), `FacebookBot`, Microsoft `Copilot` and `AliyunBot` — no
+ *     vendor documents any of them.
+ *   - Added `Google-GeminiNotebook`, Google's 2026-07-16 rename of
+ *     `Google-NotebookLM`; the old token is honoured only until August 2026.
+ *     BOTH are listed on purpose — stale traffic still carries the old one.
+ *   - Added `MistralAI-Training`, documented on Mistral's crawler page.
+ *   - Recategorised `GoogleOther` (training → ai_crawler; Google states it
+ *     "doesn't affect any specific product") and `Bytespider` (training →
+ *     search_index; ByteDance documents only a Toutiao Search index crawler).
+ *
+ * DELIBERATE EXCLUSIONS: `Google-Extended` and `Applebot-Extended` are
+ * robots.txt control tokens, not crawlers — they never appear as a live
+ * user agent, so listing them here would be dead code that can never fire.
+ *
+ * ⚠️ THE EXCLUSION ONLY ACTUALLY HOLDS FOR `Google-Extended`. Nothing matches
+ * it, so it is never reported. `Applebot-Extended` IS reported, by two
+ * independent paths:
+ *   1. Apple's UA strings carry `+http://www.apple.com/go/applebot`, whose
+ *      `applebot` substring is a TIER-1 exact token match (anchored between
+ *      `/` and `)`), so the report lands as agent `Applebot`, training.
+ *   2. The bare string `Applebot-Extended` hits the tier-2 alias
+ *      `{ alias: "applebot" }` below, because prefix anchoring accepts a
+ *      trailing `-`. That yields provider `apple`, agent null, `ai_crawler`.
+ * Neither is a bug — attributing an Apple fetch to Apple is right, and this
+ * file is a cost filter, not the classifier. Do NOT write, in code comments
+ * or in customer copy, that these tokens are "never reported" — only
+ * `Google-Extended` qualifies.
+ *
+ * All tokens are stored lowercase; matching lowercases the UA once.
+ */
+
+import type { CrawlerCategory } from "./types.js";
+
+export interface AgentToken {
+  /** Lowercased agent token, matched with token-boundary anchoring. */
+  readonly token: string;
+  /** Canonical display name of the agent (original casing). */
+  readonly agent: string;
+  readonly provider: string;
+  readonly category: CrawlerCategory;
+}
+
+/** Tier 1 — exact agent tokens. First match wins. */
+export const AGENT_TOKENS: readonly AgentToken[] = [
+  // ── answer_fetch · user-triggered live fetches ─────────────────────────
+  { token: "chatgpt-user", agent: "ChatGPT-User", provider: "openai", category: "answer_fetch" },
+  { token: "claude-user", agent: "Claude-User", provider: "anthropic", category: "answer_fetch" },
+  {
+    token: "perplexity-user",
+    agent: "Perplexity-User",
+    provider: "perplexity",
+    category: "answer_fetch",
+  },
+  { token: "google-agent", agent: "Google-Agent", provider: "google", category: "answer_fetch" },
+  {
+    token: "google-gemininotebook",
+    agent: "Google-GeminiNotebook",
+    provider: "google",
+    category: "answer_fetch",
+  },
+  // Superseded by Google-GeminiNotebook on 2026-07-16, honoured until August
+  // 2026. Kept: traffic already in flight still carries it.
+  {
+    token: "google-notebooklm",
+    agent: "Google-NotebookLM",
+    provider: "google",
+    category: "answer_fetch",
+  },
+  {
+    token: "google-read-aloud",
+    agent: "Google-Read-Aloud",
+    provider: "google",
+    category: "answer_fetch",
+  },
+  {
+    token: "mistralai-user",
+    agent: "MistralAI-User",
+    provider: "mistral",
+    category: "answer_fetch",
+  },
+  { token: "amzn-user", agent: "Amzn-User", provider: "amazon", category: "answer_fetch" },
+  {
+    token: "duckassistbot",
+    agent: "DuckAssistBot",
+    provider: "duckduckgo",
+    category: "answer_fetch",
+  },
+  {
+    token: "meta-externalfetcher",
+    agent: "meta-externalfetcher",
+    provider: "meta",
+    category: "answer_fetch",
+  },
+  { token: "kimi-user", agent: "Kimi-User", provider: "moonshot", category: "answer_fetch" },
+  { token: "xai-searchbot", agent: "xAI-SearchBot", provider: "xai", category: "answer_fetch" },
+  { token: "grok-deepsearch", agent: "Grok-DeepSearch", provider: "xai", category: "answer_fetch" },
+  { token: "qwen-user", agent: "Qwen-User", provider: "alibaba", category: "answer_fetch" },
+
+  // ── search_index · AI search index crawlers ────────────────────────────
+  { token: "oai-searchbot", agent: "OAI-SearchBot", provider: "openai", category: "search_index" },
+  {
+    token: "claude-searchbot",
+    agent: "Claude-SearchBot",
+    provider: "anthropic",
+    category: "search_index",
+  },
+  {
+    token: "perplexitybot",
+    agent: "PerplexityBot",
+    provider: "perplexity",
+    category: "search_index",
+  },
+  { token: "googlebot", agent: "Googlebot", provider: "google", category: "search_index" },
+  {
+    token: "google-inspectiontool",
+    agent: "Google-InspectionTool",
+    provider: "google",
+    category: "search_index",
+  },
+  {
+    token: "mistralai-index",
+    agent: "MistralAI-Index",
+    provider: "mistral",
+    category: "search_index",
+  },
+  { token: "bingbot", agent: "Bingbot", provider: "microsoft", category: "search_index" },
+  { token: "msnbot", agent: "msnbot", provider: "microsoft", category: "search_index" },
+  {
+    token: "amzn-searchbot",
+    agent: "Amzn-SearchBot",
+    provider: "amazon",
+    category: "search_index",
+  },
+  {
+    token: "meta-webindexer",
+    agent: "meta-webindexer",
+    provider: "meta",
+    category: "search_index",
+  },
+  {
+    token: "kimi-searchbot",
+    agent: "Kimi-SearchBot",
+    provider: "moonshot",
+    category: "search_index",
+  },
+  { token: "tiktokspider", agent: "TikTokSpider", provider: "bytedance", category: "search_index" },
+  { token: "bytespider", agent: "Bytespider", provider: "bytedance", category: "search_index" },
+  { token: "baiduspider", agent: "Baiduspider", provider: "baidu", category: "search_index" },
+  { token: "youbot", agent: "YouBot", provider: "youcom", category: "search_index" },
+
+  // ── training · corpus crawlers ─────────────────────────────────────────
+  { token: "gptbot", agent: "GPTBot", provider: "openai", category: "training" },
+  { token: "claudebot", agent: "ClaudeBot", provider: "anthropic", category: "training" },
+  {
+    token: "mistralai-training",
+    agent: "MistralAI-Training",
+    provider: "mistral",
+    category: "training",
+  },
+  {
+    token: "google-cloudvertexbot",
+    agent: "Google-CloudVertexBot",
+    provider: "google",
+    category: "training",
+  },
+  { token: "applebot", agent: "Applebot", provider: "apple", category: "training" },
+  { token: "amazonbot", agent: "Amazonbot", provider: "amazon", category: "training" },
+  {
+    token: "meta-externalagent",
+    agent: "meta-externalagent",
+    provider: "meta",
+    category: "training",
+  },
+  { token: "kimibot", agent: "KimiBot", provider: "moonshot", category: "training" },
+  { token: "ccbot", agent: "CCBot", provider: "commoncrawl", category: "training" },
+  { token: "erniebot", agent: "ERNIEBot", provider: "baidu", category: "training" },
+  { token: "qwenbot", agent: "QwenBot", provider: "alibaba", category: "training" },
+  { token: "chatglm-spider", agent: "ChatGLM-Spider", provider: "zhipu", category: "training" },
+  { token: "deepseekbot", agent: "DeepSeekBot", provider: "deepseek", category: "training" },
+  { token: "cohere-ai", agent: "cohere-ai", provider: "cohere", category: "training" },
+  {
+    token: "cohere-training-data-crawler",
+    agent: "cohere-training-data-crawler",
+    provider: "cohere",
+    category: "training",
+  },
+  { token: "ai2bot", agent: "AI2Bot", provider: "allenai", category: "training" },
+
+  // ── ai_crawler · AI-adjacent, uncategorised ────────────────────────────
+  { token: "googleother", agent: "GoogleOther", provider: "google", category: "ai_crawler" },
+  {
+    token: "meta-externalads",
+    agent: "meta-externalads",
+    provider: "meta",
+    category: "ai_crawler",
+  },
+  {
+    token: "facebookexternalhit",
+    agent: "facebookexternalhit",
+    provider: "meta",
+    category: "ai_crawler",
+  },
+  { token: "oai-adsbot", agent: "OAI-AdsBot", provider: "openai", category: "ai_crawler" },
+  { token: "grokbot", agent: "GrokBot", provider: "xai", category: "ai_crawler" },
+  { token: "xai-bot", agent: "xAI-Bot", provider: "xai", category: "ai_crawler" },
+  { token: "xai-grok", agent: "xAI-Grok", provider: "xai", category: "ai_crawler" },
+  { token: "xai-web-crawler", agent: "xAI-Web-Crawler", provider: "xai", category: "ai_crawler" },
+  { token: "grok", agent: "Grok", provider: "xai", category: "ai_crawler" },
+  { token: "doubaobot", agent: "Doubaobot", provider: "bytedance", category: "ai_crawler" },
+  { token: "yiyanbot", agent: "YiyanBot", provider: "baidu", category: "ai_crawler" },
+  { token: "tongyibot", agent: "TongyiBot", provider: "alibaba", category: "ai_crawler" },
+];
+
+export interface ProviderAlias {
+  /** Lowercased alias, matched as a LEFT-anchored prefix at a token boundary. */
+  readonly alias: string;
+  readonly provider: string;
+}
+
+/**
+ * Tier 2 — coarse provider aliases, so a NEW agent from a known vendor
+ * (e.g. a hypothetical `ChatGPT-Reader/1.0`) is still captured without a
+ * package release. Left-boundary-anchored prefix match: the alias must start
+ * at the beginning of a token (start of string or after a non-token
+ * character), which is what defeats `not-really-GPTBot`-style spoofs while
+ * still catching `Claude-NewAgent/1.0`.
+ *
+ * Tier-2 matches are reported with category `ai_crawler` locally (the server
+ * assigns the real category) and are gated by `disableOtherCrawlers`.
+ */
+export const PROVIDER_ALIASES: readonly ProviderAlias[] = [
+  { alias: "gptbot", provider: "openai" },
+  { alias: "chatgpt", provider: "openai" },
+  { alias: "oai-", provider: "openai" },
+  { alias: "claudebot", provider: "anthropic" },
+  { alias: "claude-", provider: "anthropic" },
+  { alias: "anthropic", provider: "anthropic" },
+  { alias: "perplexity", provider: "perplexity" },
+  { alias: "mistralai", provider: "mistral" },
+  { alias: "duckassist", provider: "duckduckgo" },
+  { alias: "kimi", provider: "moonshot" },
+  { alias: "grok", provider: "xai" },
+  { alias: "xai-", provider: "xai" },
+  { alias: "qwen", provider: "alibaba" },
+  { alias: "tongyi", provider: "alibaba" },
+  { alias: "bytespider", provider: "bytedance" },
+  { alias: "doubao", provider: "bytedance" },
+  { alias: "baiduspider", provider: "baidu" },
+  { alias: "erniebot", provider: "baidu" },
+  { alias: "chatglm", provider: "zhipu" },
+  { alias: "deepseek", provider: "deepseek" },
+  { alias: "cohere", provider: "cohere" },
+  { alias: "ai2bot", provider: "allenai" },
+  { alias: "ccbot", provider: "commoncrawl" },
+  { alias: "youbot", provider: "youcom" },
+  { alias: "meta-", provider: "meta" },
+  { alias: "applebot", provider: "apple" },
+  { alias: "amazonbot", provider: "amazon" },
+  { alias: "amzn-", provider: "amazon" },
+];

package/src/filter.ts

@@ -0,0 +1,178 @@
+/**
+ * Local pre-filter — everything here runs BEFORE any network call, so the
+ * overwhelming majority of requests (assets, API calls, browser subresource
+ * fetches) cost nothing.
+ *
+ * Order of checks:
+ *   1. Method gate (GET/HEAD by default)
+ *   2. sec-fetch-dest deny-list (browser subresource fetches; crawlers
+ *      don't send the header, so absence passes)
+ *   3. Crawler-facing path re-allow — /robots.txt, /llms.txt,
+ *      /llms-full.txt, *sitemap*.xml skip BOTH deny-lists below. A GPTBot
+ *      hit on /llms.txt is one of the highest-signal events available and a
+ *      naive `.txt` rule would eat it.
+ *   4. Path-prefix deny-list (extendable, never replaceable)
+ *   5. Static-extension deny-list (extendable, never replaceable)
+ */
+
+import type { ResolvedAiCrawlConfig } from "./types.js";
+
+/** Built-in path-prefix deny-list (plain prefix match on the pathname). */
+export const DEFAULT_DENY_PATH_PREFIXES: readonly string[] = [
+  "/api",
+  "/_next",
+  "/_nuxt",
+  "/_astro",
+  "/static",
+  "/assets",
+  "/public",
+  "/images",
+  "/img",
+  "/fonts",
+  "/favicon",
+  "/build",
+  "/dist",
+  "/admin",
+  "/webhook",
+  "/webhooks",
+  "/cdn-cgi",
+  "/.well-known",
+];
+
+/** Built-in static-extension deny-list (leading dot, lowercase). */
+export const DEFAULT_DENY_EXTENSIONS: readonly string[] = [
+  ".js",
+  ".mjs",
+  ".cjs",
+  ".css",
+  ".map",
+  ".json",
+  ".xml",
+  ".txt",
+  ".ico",
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".webp",
+  ".avif",
+  ".svg",
+  ".bmp",
+  ".tiff",
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".otf",
+  ".eot",
+  ".mp4",
+  ".webm",
+  ".avi",
+  ".mov",
+  ".mp3",
+  ".wav",
+  ".ogg",
+  ".flac",
+  ".zip",
+  ".gz",
+  ".tar",
+  ".rar",
+  ".7z",
+  ".wasm",
+];
+
+/** `sec-fetch-dest` values that identify browser subresource fetches. */
+export const DENY_SEC_FETCH_DESTS: readonly string[] = [
+  "audio",
+  "embed",
+  "font",
+  "image",
+  "manifest",
+  "object",
+  "script",
+  "style",
+  "track",
+  "video",
+  "worker",
+];
+
+/**
+ * Crawler-facing paths re-allowed PAST both deny-lists. Exact matches plus
+ * the `*sitemap*.xml` pattern.
+ */
+const CRAWLER_PATH_EXACT: readonly string[] = ["/robots.txt", "/llms.txt", "/llms-full.txt"];
+
+/** True for /robots.txt, /llms.txt, /llms-full.txt and *sitemap*.xml paths. */
+export function isCrawlerFacingPath(pathname: string): boolean {
+  if (CRAWLER_PATH_EXACT.includes(pathname)) return true;
+  return pathname.endsWith(".xml") && pathname.includes("sitemap");
+}
+
+/** Extract a lowercase pathname from an absolute URL or a bare path. Never throws. */
+export function extractPathname(url: string): string {
+  try {
+    let path = url;
+    // Strip scheme://host for absolute URLs without allocating a URL object.
+    const schemeIdx = path.indexOf("://");
+    if (schemeIdx !== -1) {
+      const pathStart = path.indexOf("/", schemeIdx + 3);
+      path = pathStart === -1 ? "/" : path.slice(pathStart);
+    }
+    const q = path.indexOf("?");
+    if (q !== -1) path = path.slice(0, q);
+    const h = path.indexOf("#");
+    if (h !== -1) path = path.slice(0, h);
+    if (path === "") path = "/";
+    return path.toLowerCase();
+  } catch {
+    return "/";
+  }
+}
+
+/** Last-segment extension (with dot, lowercase), or `null`. */
+function pathExtension(pathname: string): string | null {
+  const lastSlash = pathname.lastIndexOf("/");
+  const lastDot = pathname.lastIndexOf(".");
+  if (lastDot === -1 || lastDot < lastSlash || lastDot === pathname.length - 1) {
+    return null;
+  }
+  return pathname.slice(lastDot);
+}
+
+/**
+ * The full local pre-filter. Returns `true` when the request should proceed
+ * to UA matching. Never throws.
+ */
+export function passesPreFilter(
+  cfg: ResolvedAiCrawlConfig,
+  method: string | null | undefined,
+  url: string | null | undefined,
+  secFetchDest: string | null | undefined,
+): boolean {
+  try {
+    if (!method || !url) return false;
+    if (!cfg.allowedMethods.includes(method.toUpperCase())) return false;
+
+    if (secFetchDest && DENY_SEC_FETCH_DESTS.includes(secFetchDest.toLowerCase())) {
+      return false;
+    }
+
+    const pathname = extractPathname(url);
+
+    // Crawler-facing paths skip both deny-lists.
+    if (isCrawlerFacingPath(pathname)) return true;
+
+    // Segment-boundary prefix match: `/api` denies `/api` and `/api/users`
+    // but NOT `/apidocs-overview`; `/public` does not eat `/publications`.
+    // A plain startsWith here silently unreports real crawls on real paths.
+    for (const prefix of cfg.denyPathPrefixes) {
+      if (pathname === prefix || pathname.startsWith(prefix + "/")) return false;
+    }
+
+    const ext = pathExtension(pathname);
+    if (ext !== null && cfg.denyExtensions.includes(ext)) return false;
+
+    return true;
+  } catch {
+    return false;
+  }
+}