@traceten/ai-crawl 0.1.0

package/dist/crawlers.js

@@ -0,0 +1,248 @@
+/**
+ * Local crawler token data — a deliberately-loose COST FILTER, not the source
+ * of truth. Traceten's server-side crawler registry decides provider,
+ * category, verification and confidence; this list only decides "worth
+ * reporting". Keep it small and keep every token anchored against current
+ * vendor documentation.
+ *
+ * Recent reconciliation notes, kept because they explain non-obvious entries:
+ *   - Deleted `GoogleAgent` (a hyphenation typo of the real `Google-Agent`,
+ *     which stays), `FacebookBot`, Microsoft `Copilot` and `AliyunBot` — no
+ *     vendor documents any of them.
+ *   - Added `Google-GeminiNotebook`, Google's 2026-07-16 rename of
+ *     `Google-NotebookLM`; the old token is honoured only until August 2026.
+ *     BOTH are listed on purpose — stale traffic still carries the old one.
+ *   - Added `MistralAI-Training`, documented on Mistral's crawler page.
+ *   - Recategorised `GoogleOther` (training → ai_crawler; Google states it
+ *     "doesn't affect any specific product") and `Bytespider` (training →
+ *     search_index; ByteDance documents only a Toutiao Search index crawler).
+ *
+ * DELIBERATE EXCLUSIONS: `Google-Extended` and `Applebot-Extended` are
+ * robots.txt control tokens, not crawlers — they never appear as a live
+ * user agent, so listing them here would be dead code that can never fire.
+ *
+ * ⚠️ THE EXCLUSION ONLY ACTUALLY HOLDS FOR `Google-Extended`. Nothing matches
+ * it, so it is never reported. `Applebot-Extended` IS reported, by two
+ * independent paths:
+ *   1. Apple's UA strings carry `+http://www.apple.com/go/applebot`, whose
+ *      `applebot` substring is a TIER-1 exact token match (anchored between
+ *      `/` and `)`), so the report lands as agent `Applebot`, training.
+ *   2. The bare string `Applebot-Extended` hits the tier-2 alias
+ *      `{ alias: "applebot" }` below, because prefix anchoring accepts a
+ *      trailing `-`. That yields provider `apple`, agent null, `ai_crawler`.
+ * Neither is a bug — attributing an Apple fetch to Apple is right, and this
+ * file is a cost filter, not the classifier. Do NOT write, in code comments
+ * or in customer copy, that these tokens are "never reported" — only
+ * `Google-Extended` qualifies.
+ *
+ * All tokens are stored lowercase; matching lowercases the UA once.
+ */
+/** Tier 1 — exact agent tokens. First match wins. */
+export const AGENT_TOKENS = [
+    // ── answer_fetch · user-triggered live fetches ─────────────────────────
+    { token: "chatgpt-user", agent: "ChatGPT-User", provider: "openai", category: "answer_fetch" },
+    { token: "claude-user", agent: "Claude-User", provider: "anthropic", category: "answer_fetch" },
+    {
+        token: "perplexity-user",
+        agent: "Perplexity-User",
+        provider: "perplexity",
+        category: "answer_fetch",
+    },
+    { token: "google-agent", agent: "Google-Agent", provider: "google", category: "answer_fetch" },
+    {
+        token: "google-gemininotebook",
+        agent: "Google-GeminiNotebook",
+        provider: "google",
+        category: "answer_fetch",
+    },
+    // Superseded by Google-GeminiNotebook on 2026-07-16, honoured until August
+    // 2026. Kept: traffic already in flight still carries it.
+    {
+        token: "google-notebooklm",
+        agent: "Google-NotebookLM",
+        provider: "google",
+        category: "answer_fetch",
+    },
+    {
+        token: "google-read-aloud",
+        agent: "Google-Read-Aloud",
+        provider: "google",
+        category: "answer_fetch",
+    },
+    {
+        token: "mistralai-user",
+        agent: "MistralAI-User",
+        provider: "mistral",
+        category: "answer_fetch",
+    },
+    { token: "amzn-user", agent: "Amzn-User", provider: "amazon", category: "answer_fetch" },
+    {
+        token: "duckassistbot",
+        agent: "DuckAssistBot",
+        provider: "duckduckgo",
+        category: "answer_fetch",
+    },
+    {
+        token: "meta-externalfetcher",
+        agent: "meta-externalfetcher",
+        provider: "meta",
+        category: "answer_fetch",
+    },
+    { token: "kimi-user", agent: "Kimi-User", provider: "moonshot", category: "answer_fetch" },
+    { token: "xai-searchbot", agent: "xAI-SearchBot", provider: "xai", category: "answer_fetch" },
+    { token: "grok-deepsearch", agent: "Grok-DeepSearch", provider: "xai", category: "answer_fetch" },
+    { token: "qwen-user", agent: "Qwen-User", provider: "alibaba", category: "answer_fetch" },
+    // ── search_index · AI search index crawlers ────────────────────────────
+    { token: "oai-searchbot", agent: "OAI-SearchBot", provider: "openai", category: "search_index" },
+    {
+        token: "claude-searchbot",
+        agent: "Claude-SearchBot",
+        provider: "anthropic",
+        category: "search_index",
+    },
+    {
+        token: "perplexitybot",
+        agent: "PerplexityBot",
+        provider: "perplexity",
+        category: "search_index",
+    },
+    { token: "googlebot", agent: "Googlebot", provider: "google", category: "search_index" },
+    {
+        token: "google-inspectiontool",
+        agent: "Google-InspectionTool",
+        provider: "google",
+        category: "search_index",
+    },
+    {
+        token: "mistralai-index",
+        agent: "MistralAI-Index",
+        provider: "mistral",
+        category: "search_index",
+    },
+    { token: "bingbot", agent: "Bingbot", provider: "microsoft", category: "search_index" },
+    { token: "msnbot", agent: "msnbot", provider: "microsoft", category: "search_index" },
+    {
+        token: "amzn-searchbot",
+        agent: "Amzn-SearchBot",
+        provider: "amazon",
+        category: "search_index",
+    },
+    {
+        token: "meta-webindexer",
+        agent: "meta-webindexer",
+        provider: "meta",
+        category: "search_index",
+    },
+    {
+        token: "kimi-searchbot",
+        agent: "Kimi-SearchBot",
+        provider: "moonshot",
+        category: "search_index",
+    },
+    { token: "tiktokspider", agent: "TikTokSpider", provider: "bytedance", category: "search_index" },
+    { token: "bytespider", agent: "Bytespider", provider: "bytedance", category: "search_index" },
+    { token: "baiduspider", agent: "Baiduspider", provider: "baidu", category: "search_index" },
+    { token: "youbot", agent: "YouBot", provider: "youcom", category: "search_index" },
+    // ── training · corpus crawlers ─────────────────────────────────────────
+    { token: "gptbot", agent: "GPTBot", provider: "openai", category: "training" },
+    { token: "claudebot", agent: "ClaudeBot", provider: "anthropic", category: "training" },
+    {
+        token: "mistralai-training",
+        agent: "MistralAI-Training",
+        provider: "mistral",
+        category: "training",
+    },
+    {
+        token: "google-cloudvertexbot",
+        agent: "Google-CloudVertexBot",
+        provider: "google",
+        category: "training",
+    },
+    { token: "applebot", agent: "Applebot", provider: "apple", category: "training" },
+    { token: "amazonbot", agent: "Amazonbot", provider: "amazon", category: "training" },
+    {
+        token: "meta-externalagent",
+        agent: "meta-externalagent",
+        provider: "meta",
+        category: "training",
+    },
+    { token: "kimibot", agent: "KimiBot", provider: "moonshot", category: "training" },
+    { token: "ccbot", agent: "CCBot", provider: "commoncrawl", category: "training" },
+    { token: "erniebot", agent: "ERNIEBot", provider: "baidu", category: "training" },
+    { token: "qwenbot", agent: "QwenBot", provider: "alibaba", category: "training" },
+    { token: "chatglm-spider", agent: "ChatGLM-Spider", provider: "zhipu", category: "training" },
+    { token: "deepseekbot", agent: "DeepSeekBot", provider: "deepseek", category: "training" },
+    { token: "cohere-ai", agent: "cohere-ai", provider: "cohere", category: "training" },
+    {
+        token: "cohere-training-data-crawler",
+        agent: "cohere-training-data-crawler",
+        provider: "cohere",
+        category: "training",
+    },
+    { token: "ai2bot", agent: "AI2Bot", provider: "allenai", category: "training" },
+    // ── ai_crawler · AI-adjacent, uncategorised ────────────────────────────
+    { token: "googleother", agent: "GoogleOther", provider: "google", category: "ai_crawler" },
+    {
+        token: "meta-externalads",
+        agent: "meta-externalads",
+        provider: "meta",
+        category: "ai_crawler",
+    },
+    {
+        token: "facebookexternalhit",
+        agent: "facebookexternalhit",
+        provider: "meta",
+        category: "ai_crawler",
+    },
+    { token: "oai-adsbot", agent: "OAI-AdsBot", provider: "openai", category: "ai_crawler" },
+    { token: "grokbot", agent: "GrokBot", provider: "xai", category: "ai_crawler" },
+    { token: "xai-bot", agent: "xAI-Bot", provider: "xai", category: "ai_crawler" },
+    { token: "xai-grok", agent: "xAI-Grok", provider: "xai", category: "ai_crawler" },
+    { token: "xai-web-crawler", agent: "xAI-Web-Crawler", provider: "xai", category: "ai_crawler" },
+    { token: "grok", agent: "Grok", provider: "xai", category: "ai_crawler" },
+    { token: "doubaobot", agent: "Doubaobot", provider: "bytedance", category: "ai_crawler" },
+    { token: "yiyanbot", agent: "YiyanBot", provider: "baidu", category: "ai_crawler" },
+    { token: "tongyibot", agent: "TongyiBot", provider: "alibaba", category: "ai_crawler" },
+];
+/**
+ * Tier 2 — coarse provider aliases, so a NEW agent from a known vendor
+ * (e.g. a hypothetical `ChatGPT-Reader/1.0`) is still captured without a
+ * package release. Left-boundary-anchored prefix match: the alias must start
+ * at the beginning of a token (start of string or after a non-token
+ * character), which is what defeats `not-really-GPTBot`-style spoofs while
+ * still catching `Claude-NewAgent/1.0`.
+ *
+ * Tier-2 matches are reported with category `ai_crawler` locally (the server
+ * assigns the real category) and are gated by `disableOtherCrawlers`.
+ */
+export const PROVIDER_ALIASES = [
+    { alias: "gptbot", provider: "openai" },
+    { alias: "chatgpt", provider: "openai" },
+    { alias: "oai-", provider: "openai" },
+    { alias: "claudebot", provider: "anthropic" },
+    { alias: "claude-", provider: "anthropic" },
+    { alias: "anthropic", provider: "anthropic" },
+    { alias: "perplexity", provider: "perplexity" },
+    { alias: "mistralai", provider: "mistral" },
+    { alias: "duckassist", provider: "duckduckgo" },
+    { alias: "kimi", provider: "moonshot" },
+    { alias: "grok", provider: "xai" },
+    { alias: "xai-", provider: "xai" },
+    { alias: "qwen", provider: "alibaba" },
+    { alias: "tongyi", provider: "alibaba" },
+    { alias: "bytespider", provider: "bytedance" },
+    { alias: "doubao", provider: "bytedance" },
+    { alias: "baiduspider", provider: "baidu" },
+    { alias: "erniebot", provider: "baidu" },
+    { alias: "chatglm", provider: "zhipu" },
+    { alias: "deepseek", provider: "deepseek" },
+    { alias: "cohere", provider: "cohere" },
+    { alias: "ai2bot", provider: "allenai" },
+    { alias: "ccbot", provider: "commoncrawl" },
+    { alias: "youbot", provider: "youcom" },
+    { alias: "meta-", provider: "meta" },
+    { alias: "applebot", provider: "apple" },
+    { alias: "amazonbot", provider: "amazon" },
+    { alias: "amzn-", provider: "amazon" },
+];
+//# sourceMappingURL=crawlers.js.map

package/dist/crawlers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawlers.js","sourceRoot":"","sources":["../src/crawlers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAaH,qDAAqD;AACrD,MAAM,CAAC,MAAM,YAAY,GAA0B;IACjD,0EAA0E;IAC1E,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC9F,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC/F;QACE,KAAK,EAAE,iBAAiB;QACxB,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC9F;QACE,KAAK,EAAE,uBAAuB;QAC9B,KAAK,EAAE,uBAAuB;QAC9B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,cAAc;KACzB;IACD,2EAA2E;IAC3E,0DAA0D;IAC1D;QACE,KAAK,EAAE,mBAAmB;QAC1B,KAAK,EAAE,mBAAmB;QAC1B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,mBAAmB;QAC1B,KAAK,EAAE,mBAAmB;QAC1B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,gBAAgB;QACvB,QAAQ,EAAE,SAAS;QACnB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IACxF;QACE,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,sBAAsB;QAC7B,KAAK,EAAE,sBAAsB;QAC7B,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC1F,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC7F,EAAE,KAAK,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE;IACjG,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE;IAEzF,0EAA0E;IAC1E,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IAChG;QACE,KAAK,EAAE,kBAAkB;QACzB,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,eAAe;QACtB,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IACxF;QACE,KAAK,EAAE,uBAAuB;QAC9B,KAAK,EAAE,uBAAuB;QAC9B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,iBAAiB;QACxB,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,SAAS;QACnB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE;IACvF,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE;IACrF;QACE,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,gBAAgB;QACvB,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,iBAAiB;QACxB,KAAK,EAAE,iBAAiB;QACxB,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,cAAc;KACzB;IACD;QACE,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,gBAAgB;QACvB,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,cAAc;KACzB;IACD,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE;IACjG,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC7F,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE;IAC3F,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE;IAElF,0EAA0E;IAC1E,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC9E,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE;IACvF;QACE,KAAK,EAAE,oBAAoB;QAC3B,KAAK,EAAE,oBAAoB;QAC3B,QAAQ,EAAE,SAAS;QACnB,QAAQ,EAAE,UAAU;KACrB;IACD;QACE,KAAK,EAAE,uBAAuB;QAC9B,KAAK,EAAE,uBAAuB;QAC9B,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,UAAU;KACrB;IACD,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;IACjF,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE;IACpF;QACE,KAAK,EAAE,oBAAoB;QAC3B,KAAK,EAAE,oBAAoB;QAC3B,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,UAAU;KACrB;IACD,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE;IAClF,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE;IACjF,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;IACjF,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE;IACjF,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC7F,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC1F,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE;IACpF;QACE,KAAK,EAAE,8BAA8B;QACrC,KAAK,EAAE,8BAA8B;QACrC,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,UAAU;KACrB;IACD,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE;IAE/E,0EAA0E;IAC1E,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC1F;QACE,KAAK,EAAE,kBAAkB;QACzB,KAAK,EAAE,kBAAkB;QACzB,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,YAAY;KACvB;IACD;QACE,KAAK,EAAE,qBAAqB;QAC5B,KAAK,EAAE,qBAAqB;QAC5B,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,YAAY;KACvB;IACD,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE;IACxF,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC/E,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC/E,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE;IACjF,EAAE,KAAK,EAAE,iBAAiB,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC/F,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE;IACzE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE;IACzF,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE;IACnF,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE;CACxF,CAAC;AAQF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACxD,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACvC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACxC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACrC,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC7C,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC3C,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC7C,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC/C,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,EAAE;IAC3C,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE;IAC/C,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE;IACvC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE;IAClC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE;IAClC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE;IACtC,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE;IACxC,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC9C,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE;IAC1C,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC3C,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE;IACxC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE;IACvC,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC3C,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACvC,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE;IACxC,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;IAC3C,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE;IACvC,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;IACpC,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE;IACxC,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE;IAC1C,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE;CACvC,CAAC"}

package/dist/filter.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Local pre-filter — everything here runs BEFORE any network call, so the
+ * overwhelming majority of requests (assets, API calls, browser subresource
+ * fetches) cost nothing.
+ *
+ * Order of checks:
+ *   1. Method gate (GET/HEAD by default)
+ *   2. sec-fetch-dest deny-list (browser subresource fetches; crawlers
+ *      don't send the header, so absence passes)
+ *   3. Crawler-facing path re-allow — /robots.txt, /llms.txt,
+ *      /llms-full.txt, *sitemap*.xml skip BOTH deny-lists below. A GPTBot
+ *      hit on /llms.txt is one of the highest-signal events available and a
+ *      naive `.txt` rule would eat it.
+ *   4. Path-prefix deny-list (extendable, never replaceable)
+ *   5. Static-extension deny-list (extendable, never replaceable)
+ */
+import type { ResolvedAiCrawlConfig } from "./types.js";
+/** Built-in path-prefix deny-list (plain prefix match on the pathname). */
+export declare const DEFAULT_DENY_PATH_PREFIXES: readonly string[];
+/** Built-in static-extension deny-list (leading dot, lowercase). */
+export declare const DEFAULT_DENY_EXTENSIONS: readonly string[];
+/** `sec-fetch-dest` values that identify browser subresource fetches. */
+export declare const DENY_SEC_FETCH_DESTS: readonly string[];
+/** True for /robots.txt, /llms.txt, /llms-full.txt and *sitemap*.xml paths. */
+export declare function isCrawlerFacingPath(pathname: string): boolean;
+/** Extract a lowercase pathname from an absolute URL or a bare path. Never throws. */
+export declare function extractPathname(url: string): string;
+/**
+ * The full local pre-filter. Returns `true` when the request should proceed
+ * to UA matching. Never throws.
+ */
+export declare function passesPreFilter(cfg: ResolvedAiCrawlConfig, method: string | null | undefined, url: string | null | undefined, secFetchDest: string | null | undefined): boolean;
+//# sourceMappingURL=filter.d.ts.map

package/dist/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../src/filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD,2EAA2E;AAC3E,eAAO,MAAM,0BAA0B,EAAE,SAAS,MAAM,EAmBvD,CAAC;AAEF,oEAAoE;AACpE,eAAO,MAAM,uBAAuB,EAAE,SAAS,MAAM,EAsCpD,CAAC;AAEF,yEAAyE;AACzE,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAYjD,CAAC;AAQF,+EAA+E;AAC/E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAG7D;AAED,sFAAsF;AACtF,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAkBnD;AAYD;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,qBAAqB,EAC1B,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EACjC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GACtC,OAAO,CA4BT"}

package/dist/filter.js

@@ -0,0 +1,169 @@
+/**
+ * Local pre-filter — everything here runs BEFORE any network call, so the
+ * overwhelming majority of requests (assets, API calls, browser subresource
+ * fetches) cost nothing.
+ *
+ * Order of checks:
+ *   1. Method gate (GET/HEAD by default)
+ *   2. sec-fetch-dest deny-list (browser subresource fetches; crawlers
+ *      don't send the header, so absence passes)
+ *   3. Crawler-facing path re-allow — /robots.txt, /llms.txt,
+ *      /llms-full.txt, *sitemap*.xml skip BOTH deny-lists below. A GPTBot
+ *      hit on /llms.txt is one of the highest-signal events available and a
+ *      naive `.txt` rule would eat it.
+ *   4. Path-prefix deny-list (extendable, never replaceable)
+ *   5. Static-extension deny-list (extendable, never replaceable)
+ */
+/** Built-in path-prefix deny-list (plain prefix match on the pathname). */
+export const DEFAULT_DENY_PATH_PREFIXES = [
+    "/api",
+    "/_next",
+    "/_nuxt",
+    "/_astro",
+    "/static",
+    "/assets",
+    "/public",
+    "/images",
+    "/img",
+    "/fonts",
+    "/favicon",
+    "/build",
+    "/dist",
+    "/admin",
+    "/webhook",
+    "/webhooks",
+    "/cdn-cgi",
+    "/.well-known",
+];
+/** Built-in static-extension deny-list (leading dot, lowercase). */
+export const DEFAULT_DENY_EXTENSIONS = [
+    ".js",
+    ".mjs",
+    ".cjs",
+    ".css",
+    ".map",
+    ".json",
+    ".xml",
+    ".txt",
+    ".ico",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".avif",
+    ".svg",
+    ".bmp",
+    ".tiff",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".otf",
+    ".eot",
+    ".mp4",
+    ".webm",
+    ".avi",
+    ".mov",
+    ".mp3",
+    ".wav",
+    ".ogg",
+    ".flac",
+    ".zip",
+    ".gz",
+    ".tar",
+    ".rar",
+    ".7z",
+    ".wasm",
+];
+/** `sec-fetch-dest` values that identify browser subresource fetches. */
+export const DENY_SEC_FETCH_DESTS = [
+    "audio",
+    "embed",
+    "font",
+    "image",
+    "manifest",
+    "object",
+    "script",
+    "style",
+    "track",
+    "video",
+    "worker",
+];
+/**
+ * Crawler-facing paths re-allowed PAST both deny-lists. Exact matches plus
+ * the `*sitemap*.xml` pattern.
+ */
+const CRAWLER_PATH_EXACT = ["/robots.txt", "/llms.txt", "/llms-full.txt"];
+/** True for /robots.txt, /llms.txt, /llms-full.txt and *sitemap*.xml paths. */
+export function isCrawlerFacingPath(pathname) {
+    if (CRAWLER_PATH_EXACT.includes(pathname))
+        return true;
+    return pathname.endsWith(".xml") && pathname.includes("sitemap");
+}
+/** Extract a lowercase pathname from an absolute URL or a bare path. Never throws. */
+export function extractPathname(url) {
+    try {
+        let path = url;
+        // Strip scheme://host for absolute URLs without allocating a URL object.
+        const schemeIdx = path.indexOf("://");
+        if (schemeIdx !== -1) {
+            const pathStart = path.indexOf("/", schemeIdx + 3);
+            path = pathStart === -1 ? "/" : path.slice(pathStart);
+        }
+        const q = path.indexOf("?");
+        if (q !== -1)
+            path = path.slice(0, q);
+        const h = path.indexOf("#");
+        if (h !== -1)
+            path = path.slice(0, h);
+        if (path === "")
+            path = "/";
+        return path.toLowerCase();
+    }
+    catch {
+        return "/";
+    }
+}
+/** Last-segment extension (with dot, lowercase), or `null`. */
+function pathExtension(pathname) {
+    const lastSlash = pathname.lastIndexOf("/");
+    const lastDot = pathname.lastIndexOf(".");
+    if (lastDot === -1 || lastDot < lastSlash || lastDot === pathname.length - 1) {
+        return null;
+    }
+    return pathname.slice(lastDot);
+}
+/**
+ * The full local pre-filter. Returns `true` when the request should proceed
+ * to UA matching. Never throws.
+ */
+export function passesPreFilter(cfg, method, url, secFetchDest) {
+    try {
+        if (!method || !url)
+            return false;
+        if (!cfg.allowedMethods.includes(method.toUpperCase()))
+            return false;
+        if (secFetchDest && DENY_SEC_FETCH_DESTS.includes(secFetchDest.toLowerCase())) {
+            return false;
+        }
+        const pathname = extractPathname(url);
+        // Crawler-facing paths skip both deny-lists.
+        if (isCrawlerFacingPath(pathname))
+            return true;
+        // Segment-boundary prefix match: `/api` denies `/api` and `/api/users`
+        // but NOT `/apidocs-overview`; `/public` does not eat `/publications`.
+        // A plain startsWith here silently unreports real crawls on real paths.
+        for (const prefix of cfg.denyPathPrefixes) {
+            if (pathname === prefix || pathname.startsWith(prefix + "/"))
+                return false;
+        }
+        const ext = pathExtension(pathname);
+        if (ext !== null && cfg.denyExtensions.includes(ext))
+            return false;
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=filter.js.map

package/dist/filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.js","sourceRoot":"","sources":["../src/filter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,2EAA2E;AAC3E,MAAM,CAAC,MAAM,0BAA0B,GAAsB;IAC3D,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,SAAS;IACT,SAAS;IACT,SAAS;IACT,SAAS;IACT,MAAM;IACN,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,UAAU;IACV,WAAW;IACX,UAAU;IACV,cAAc;CACf,CAAC;AAEF,oEAAoE;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAsB;IACxD,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,OAAO;CACR,CAAC;AAEF,yEAAyE;AACzE,MAAM,CAAC,MAAM,oBAAoB,GAAsB;IACrD,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;CACT,CAAC;AAEF;;;GAGG;AACH,MAAM,kBAAkB,GAAsB,CAAC,aAAa,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;AAE7F,+EAA+E;AAC/E,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,IAAI,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AACnE,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC;QACH,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,yEAAyE;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YACnD,IAAI,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,IAAI,IAAI,KAAK,EAAE;YAAE,IAAI,GAAG,GAAG,CAAC;QAC5B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,+DAA+D;AAC/D,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,IAAI,OAAO,KAAK,CAAC,CAAC,IAAI,OAAO,GAAG,SAAS,IAAI,OAAO,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,GAA0B,EAC1B,MAAiC,EACjC,GAA8B,EAC9B,YAAuC;IAEvC,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAErE,IAAI,YAAY,IAAI,oBAAoB,CAAC,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC9E,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QAEtC,6CAA6C;QAC7C,IAAI,mBAAmB,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QAE/C,uEAAuE;QACvE,uEAAuE;QACvE,wEAAwE;QACxE,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC7E,CAAC;QAED,MAAM,GAAG,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAEnE,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,21 @@
+/**
+ * @traceten/ai-crawl — server-side AI crawler tracking.
+ *
+ * AI crawlers (GPTBot, ClaudeBot, PerplexityBot, …) never execute
+ * JavaScript, so the browser snippet cannot see them. The observation point
+ * has to be the customer's server — this package is that observation point.
+ * It pre-filters locally, matches user agents with anchored tokens, and
+ * reports plausible AI crawls to `POST /v1/ai-crawls`. Classification,
+ * verification and confidence are decided server-side.
+ *
+ * Adapters: `@traceten/ai-crawl/next`, `/cloudflare-pages`,
+ * `/cloudflare-workers`, `/express`, `/hono`.
+ */
+export { AUTH_TOKEN_PREFIX, DEFAULT_ALLOWED_METHODS, DEFAULT_ENDPOINT, defineAiCrawlConfig, isResolvedConfig, } from "./config.js";
+export { DEFAULT_DENY_EXTENSIONS, DEFAULT_DENY_PATH_PREFIXES, DENY_SEC_FETCH_DESTS, isCrawlerFacingPath, passesPreFilter, } from "./filter.js";
+export { matchCrawler } from "./matcher.js";
+export { resolveCrawlerIp } from "./ip.js";
+export { REPORT_TIMEOUT_MS, buildPayload, sendReport, type SendOptions } from "./report.js";
+export { evaluateRequest, factsFromFetchRequest, trackFacts, type FetchLikeRequest, } from "./track.js";
+export type { AiCrawlConfig, AiCrawlDeliveryError, AiCrawlWirePayload, CrawlerCategory, CrawlerMatch, RequestFacts, ResolvedAiCrawlConfig, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,UAAU,EACV,KAAK,gBAAgB,GACtB,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,aAAa,EACb,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,qBAAqB,GACtB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,20 @@
+/**
+ * @traceten/ai-crawl — server-side AI crawler tracking.
+ *
+ * AI crawlers (GPTBot, ClaudeBot, PerplexityBot, …) never execute
+ * JavaScript, so the browser snippet cannot see them. The observation point
+ * has to be the customer's server — this package is that observation point.
+ * It pre-filters locally, matches user agents with anchored tokens, and
+ * reports plausible AI crawls to `POST /v1/ai-crawls`. Classification,
+ * verification and confidence are decided server-side.
+ *
+ * Adapters: `@traceten/ai-crawl/next`, `/cloudflare-pages`,
+ * `/cloudflare-workers`, `/express`, `/hono`.
+ */
+export { AUTH_TOKEN_PREFIX, DEFAULT_ALLOWED_METHODS, DEFAULT_ENDPOINT, defineAiCrawlConfig, isResolvedConfig, } from "./config.js";
+export { DEFAULT_DENY_EXTENSIONS, DEFAULT_DENY_PATH_PREFIXES, DENY_SEC_FETCH_DESTS, isCrawlerFacingPath, passesPreFilter, } from "./filter.js";
+export { matchCrawler } from "./matcher.js";
+export { resolveCrawlerIp } from "./ip.js";
+export { REPORT_TIMEOUT_MS, buildPayload, sendReport } from "./report.js";
+export { evaluateRequest, factsFromFetchRequest, trackFacts, } from "./track.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,aAAa,CAAC;AAC5F,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,UAAU,GAEX,MAAM,YAAY,CAAC"}

package/dist/ip.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Crawler IP resolution.
+ *
+ * The crawler's TCP connection terminates at the CUSTOMER's server, so the
+ * crawler IP is only observable here — Traceten's edge sees the customer's
+ * origin IP, not the crawler's. This module
+ * derives the best-available value; the payload OMITS the field rather than
+ * send a wrong one.
+ *
+ * Resolution order:
+ *   1. `cf-connecting-ip` — ONLY when `trustCfConnectingIp` is set. The
+ *      Cloudflare adapters set it automatically (the platform strips and
+ *      rewrites the header there); anywhere else it is client-forgeable,
+ *      and a forged vendor-range IP paired with a vendor UA is exactly the
+ *      spoof the server-side verification exists to catch.
+ *   2. `x-forwarded-for` — ONLY when `trustProxy` is true, selecting the
+ *      entry `proxyDepth` hops from the right (each trusted proxy appends
+ *      one entry; anything further left is attacker-controllable)
+ *   3. Socket remote address, where the runtime exposes one
+ */
+import type { RequestFacts, ResolvedAiCrawlConfig } from "./types.js";
+/** Normalise a candidate; returns `undefined` when it is not a plausible IP. */
+export declare function normalizeIp(raw: string | null | undefined): string | undefined;
+/**
+ * Select the client entry from an `x-forwarded-for` value given the number
+ * of trusted proxies that append to it. Returns `undefined` when the chain
+ * is shorter than `proxyDepth` (a wrong value is worse than none).
+ */
+export declare function ipFromForwardedFor(headerValue: string, proxyDepth: number): string | undefined;
+/**
+ * Resolve the crawler IP from a request. Returns `undefined` when no
+ * trustworthy value is derivable. Never throws.
+ */
+export declare function resolveCrawlerIp(cfg: ResolvedAiCrawlConfig, facts: RequestFacts): string | undefined;
+//# sourceMappingURL=ip.d.ts.map

package/dist/ip.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ip.d.ts","sourceRoot":"","sources":["../src/ip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAQtE,gFAAgF;AAChF,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAqC9E;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAS9F;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,qBAAqB,EAC1B,KAAK,EAAE,YAAY,GAClB,MAAM,GAAG,SAAS,CAmBpB"}