@vercel/agent-readability 0.1.0

package/dist/index.cjs

@@ -0,0 +1,175 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  AI_AGENT_UA_PATTERNS: () => AI_AGENT_UA_PATTERNS,
+  BOT_LIKE_REGEX: () => BOT_LIKE_REGEX,
+  SIGNATURE_AGENT_DOMAINS: () => SIGNATURE_AGENT_DOMAINS,
+  TRADITIONAL_BOT_PATTERNS: () => TRADITIONAL_BOT_PATTERNS,
+  acceptsMarkdown: () => acceptsMarkdown,
+  generateNotFoundMarkdown: () => generateNotFoundMarkdown,
+  isAIAgent: () => isAIAgent,
+  shouldServeMarkdown: () => shouldServeMarkdown
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/patterns.ts
+var AI_AGENT_UA_PATTERNS = [
+  // Anthropic — https://support.claude.com/en/articles/8896518
+  "claudebot",
+  "claude-searchbot",
+  "claude-user",
+  "anthropic-ai",
+  "claude-web",
+  // OpenAI — https://platform.openai.com/docs/bots
+  "chatgpt",
+  "gptbot",
+  "oai-searchbot",
+  "openai",
+  // Google AI
+  "gemini",
+  "bard",
+  "google-cloudvertexbot",
+  "google-extended",
+  // Meta
+  "meta-externalagent",
+  "meta-externalfetcher",
+  "meta-webindexer",
+  // Search/Research AI
+  "perplexity",
+  "youbot",
+  "you.com",
+  "deepseekbot",
+  // Coding assistants
+  "cursor",
+  "github-copilot",
+  "codeium",
+  "tabnine",
+  "sourcegraph",
+  // Other AI agents / data scrapers
+  "cohere-ai",
+  "bytespider",
+  "amazonbot",
+  "ai2bot",
+  "diffbot",
+  "omgili",
+  "omgilibot"
+];
+var SIGNATURE_AGENT_DOMAINS = ["chatgpt.com"];
+var TRADITIONAL_BOT_PATTERNS = [
+  "googlebot",
+  "bingbot",
+  "yandexbot",
+  "baiduspider",
+  "duckduckbot",
+  "slurp",
+  "msnbot",
+  "facebot",
+  "twitterbot",
+  "linkedinbot",
+  "whatsapp",
+  "telegrambot",
+  "pingdom",
+  "uptimerobot",
+  "newrelic",
+  "datadog",
+  "statuspage",
+  "site24x7",
+  "applebot"
+];
+var BOT_LIKE_REGEX = /bot|agent|fetch|crawl|spider|search/i;
+
+// src/detection.ts
+function isAIAgent(request) {
+  const userAgent = request.headers.get("user-agent");
+  const lowerUA = userAgent?.toLowerCase() ?? "";
+  if (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {
+    return { detected: true, method: "ua-match" };
+  }
+  const signatureAgent = request.headers.get("signature-agent");
+  if (signatureAgent) {
+    const lowerSig = signatureAgent.toLowerCase();
+    if (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {
+      return { detected: true, method: "signature-agent" };
+    }
+  }
+  const secFetchMode = request.headers.get("sec-fetch-mode");
+  if (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {
+    const isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));
+    if (!isTraditionalBot) {
+      return { detected: true, method: "heuristic" };
+    }
+  }
+  return { detected: false, method: null };
+}
+
+// src/negotiation.ts
+var DEFAULT_MARKDOWN_TYPES = ["text/markdown", "text/x-markdown"];
+function acceptsMarkdown(request, options) {
+  const accept = request.headers.get("accept");
+  if (!accept) return false;
+  const types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;
+  const lowerAccept = accept.toLowerCase();
+  return types.some((type) => lowerAccept.includes(type));
+}
+function shouldServeMarkdown(request, options) {
+  const detection = isAIAgent(request);
+  if (detection.detected) {
+    return { serve: true, reason: "agent", detection };
+  }
+  if (acceptsMarkdown(request, options)) {
+    return { serve: true, reason: "accept-header", detection };
+  }
+  return { serve: false, reason: null, detection };
+}
+
+// src/not-found.ts
+function generateNotFoundMarkdown(path, options) {
+  const sitemap = options?.sitemapUrl ?? "/sitemap.md";
+  const index = options?.indexUrl ?? "/llms.txt";
+  const base = options?.baseUrl ?? "";
+  return `# Page Not Found
+
+The URL \`${path}\` does not exist.
+
+## How to find the correct page
+
+1. **Browse the sitemap**: [${sitemap}](${base}${sitemap}) - A structured index of all pages
+2. **Browse the full index**: [${index}](${base}${index}) - Complete documentation index
+
+## Tips for requesting documentation
+
+- For markdown responses, append \`.md\` to URLs (e.g., \`/docs/functions.md\`)
+- Use \`Accept: text/markdown\` header for content negotiation
+`;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AI_AGENT_UA_PATTERNS,
+  BOT_LIKE_REGEX,
+  SIGNATURE_AGENT_DOMAINS,
+  TRADITIONAL_BOT_PATTERNS,
+  acceptsMarkdown,
+  generateNotFoundMarkdown,
+  isAIAgent,
+  shouldServeMarkdown
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/patterns.ts","../src/detection.ts","../src/negotiation.ts","../src/not-found.ts"],"sourcesContent":["export { isAIAgent } from \"./detection\";\nexport { acceptsMarkdown, shouldServeMarkdown } from \"./negotiation\";\nexport { generateNotFoundMarkdown } from \"./not-found\";\nexport {\n\tAI_AGENT_UA_PATTERNS,\n\tBOT_LIKE_REGEX,\n\tSIGNATURE_AGENT_DOMAINS,\n\tTRADITIONAL_BOT_PATTERNS,\n} from \"./patterns\";\nexport type {\n\tDetectionMethod,\n\tDetectionResult,\n\tMinimalRequest,\n} from \"./types\";\nexport type { NotFoundOptions } from \"./not-found\";\nexport type { AcceptMarkdownOptions, ShouldServeMarkdownResult } from \"./negotiation\";\n","/**\n * Layer 1: Known AI agent UA substrings (lowercase).\n * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.\n * Last reviewed: 2026-03-20\n */\nexport const AI_AGENT_UA_PATTERNS: readonly string[] = [\n\t// Anthropic — https://support.claude.com/en/articles/8896518\n\t\"claudebot\",\n\t\"claude-searchbot\",\n\t\"claude-user\",\n\t\"anthropic-ai\",\n\t\"claude-web\",\n\n\t// OpenAI — https://platform.openai.com/docs/bots\n\t\"chatgpt\",\n\t\"gptbot\",\n\t\"oai-searchbot\",\n\t\"openai\",\n\n\t// Google AI\n\t\"gemini\",\n\t\"bard\",\n\t\"google-cloudvertexbot\",\n\t\"google-extended\",\n\n\t// Meta\n\t\"meta-externalagent\",\n\t\"meta-externalfetcher\",\n\t\"meta-webindexer\",\n\n\t// Search/Research AI\n\t\"perplexity\",\n\t\"youbot\",\n\t\"you.com\",\n\t\"deepseekbot\",\n\n\t// Coding assistants\n\t\"cursor\",\n\t\"github-copilot\",\n\t\"codeium\",\n\t\"tabnine\",\n\t\"sourcegraph\",\n\n\t// Other AI agents / data scrapers\n\t\"cohere-ai\",\n\t\"bytespider\",\n\t\"amazonbot\",\n\t\"ai2bot\",\n\t\"diffbot\",\n\t\"omgili\",\n\t\"omgilibot\",\n];\n\n/**\n * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).\n */\nexport const SIGNATURE_AGENT_DOMAINS: readonly string[] = [\"chatgpt.com\"];\n\n/**\n * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the\n * heuristic layer (search engine crawlers, social previews, monitoring tools).\n */\nexport const TRADITIONAL_BOT_PATTERNS: readonly string[] = [\n\t\"googlebot\",\n\t\"bingbot\",\n\t\"yandexbot\",\n\t\"baiduspider\",\n\t\"duckduckbot\",\n\t\"slurp\",\n\t\"msnbot\",\n\t\"facebot\",\n\t\"twitterbot\",\n\t\"linkedinbot\",\n\t\"whatsapp\",\n\t\"telegrambot\",\n\t\"pingdom\",\n\t\"uptimerobot\",\n\t\"newrelic\",\n\t\"datadog\",\n\t\"statuspage\",\n\t\"site24x7\",\n\t\"applebot\",\n];\n\n/**\n * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).\n * No word boundaries — keywords commonly appear in compound names.\n */\nexport const BOT_LIKE_REGEX: RegExp = /bot|agent|fetch|crawl|spider|search/i;\n","import {\n\tAI_AGENT_UA_PATTERNS,\n\tBOT_LIKE_REGEX,\n\tSIGNATURE_AGENT_DOMAINS,\n\tTRADITIONAL_BOT_PATTERNS,\n} from \"./patterns\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\n/**\n * Detects AI agents from HTTP request headers.\n *\n * Three detection layers (checked in order):\n * 1. Known UA patterns (definitive)\n * 2. Signature-Agent header (definitive, RFC 9421)\n * 3. Missing sec-fetch-mode heuristic (catches unknown bots)\n *\n * Optimizes for recall over precision: serving markdown to a non-AI bot\n * is low-harm; missing an AI agent means a worse experience.\n */\nexport function isAIAgent(request: MinimalRequest): DetectionResult {\n\tconst userAgent = request.headers.get(\"user-agent\");\n\tconst lowerUA = userAgent?.toLowerCase() ?? \"\";\n\n\t// Layer 1: Known UA pattern match\n\tif (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {\n\t\treturn { detected: true, method: \"ua-match\" };\n\t}\n\n\t// Layer 2: Signature-Agent header (RFC 9421, used by ChatGPT agent)\n\tconst signatureAgent = request.headers.get(\"signature-agent\");\n\tif (signatureAgent) {\n\t\tconst lowerSig = signatureAgent.toLowerCase();\n\t\tif (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {\n\t\t\treturn { detected: true, method: \"signature-agent\" };\n\t\t}\n\t}\n\n\t// Layer 3: Missing browser fingerprint heuristic\n\t// Real browsers (Chrome 76+, Firefox 90+, Safari 16.4+) send sec-fetch-mode\n\t// on navigation requests. Its absence signals a programmatic client.\n\tconst secFetchMode = request.headers.get(\"sec-fetch-mode\");\n\tif (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {\n\t\tconst isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));\n\t\tif (!isTraditionalBot) {\n\t\t\treturn { detected: true, method: \"heuristic\" };\n\t\t}\n\t}\n\n\treturn { detected: false, method: null };\n}\n","import { isAIAgent } from \"./detection\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\nconst DEFAULT_MARKDOWN_TYPES = [\"text/markdown\", \"text/x-markdown\"];\n\nexport interface AcceptMarkdownOptions {\n\tmediaTypes?: string[];\n}\n\n/**\n * Check if the request prefers markdown via the Accept header.\n */\nexport function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean {\n\tconst accept = request.headers.get(\"accept\");\n\tif (!accept) return false;\n\n\tconst types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;\n\tconst lowerAccept = accept.toLowerCase();\n\treturn types.some((type) => lowerAccept.includes(type));\n}\n\nexport interface ShouldServeMarkdownResult {\n\tserve: boolean;\n\treason: \"agent\" | \"accept-header\" | null;\n\tdetection: DetectionResult;\n}\n\n/**\n * Combines agent detection and content negotiation into one call.\n * Returns whether to serve markdown and why.\n */\nexport function shouldServeMarkdown(\n\trequest: MinimalRequest,\n\toptions?: AcceptMarkdownOptions,\n): ShouldServeMarkdownResult {\n\tconst detection = isAIAgent(request);\n\tif (detection.detected) {\n\t\treturn { serve: true, reason: \"agent\", detection };\n\t}\n\n\tif (acceptsMarkdown(request, options)) {\n\t\treturn { serve: true, reason: \"accept-header\", detection };\n\t}\n\n\treturn { serve: false, reason: null, detection };\n}\n","export interface NotFoundOptions {\n\tsitemapUrl?: string;\n\tindexUrl?: string;\n\tbaseUrl?: string;\n}\n\n/**\n * Generates a markdown body for missing pages with links to discovery endpoints.\n * Return with a 200 status (agents discard 404 response bodies).\n */\nexport function generateNotFoundMarkdown(path: string, options?: NotFoundOptions): string {\n\tconst sitemap = options?.sitemapUrl ?? \"/sitemap.md\";\n\tconst index = options?.indexUrl ?? \"/llms.txt\";\n\tconst base = options?.baseUrl ?? \"\";\n\n\treturn `# Page Not Found\n\nThe URL \\`${path}\\` does not exist.\n\n## How to find the correct page\n\n1. **Browse the sitemap**: [${sitemap}](${base}${sitemap}) - A structured index of all pages\n2. **Browse the full index**: [${index}](${base}${index}) - Complete documentation index\n\n## Tips for requesting documentation\n\n- For markdown responses, append \\`.md\\` to URLs (e.g., \\`/docs/functions.md\\`)\n- Use \\`Accept: text/markdown\\` header for content negotiation\n`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACKO,IAAM,uBAA0C;AAAA;AAAA,EAEtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAKO,IAAM,0BAA6C,CAAC,aAAa;AAMjE,IAAM,2BAA8C;AAAA,EAC1D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAMO,IAAM,iBAAyB;;;ACrE/B,SAAS,UAAU,SAA0C;AACnE,QAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY;AAClD,QAAM,UAAU,WAAW,YAAY,KAAK;AAG5C,MAAI,WAAW,qBAAqB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC,GAAG;AACjF,WAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAAA,EAC7C;AAGA,QAAM,iBAAiB,QAAQ,QAAQ,IAAI,iBAAiB;AAC5D,MAAI,gBAAgB;AACnB,UAAM,WAAW,eAAe,YAAY;AAC5C,QAAI,wBAAwB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC,GAAG;AACxE,aAAO,EAAE,UAAU,MAAM,QAAQ,kBAAkB;AAAA,IACpD;AAAA,EACD;AAKA,QAAM,eAAe,QAAQ,QAAQ,IAAI,gBAAgB;AACzD,MAAI,CAAC,gBAAgB,WAAW,eAAe,KAAK,OAAO,GAAG;AAC7D,UAAM,mBAAmB,yBAAyB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC;AAC7F,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,UAAU,MAAM,QAAQ,YAAY;AAAA,IAC9C;AAAA,EACD;AAEA,SAAO,EAAE,UAAU,OAAO,QAAQ,KAAK;AACxC;;;AC9CA,IAAM,yBAAyB,CAAC,iBAAiB,iBAAiB;AAS3D,SAAS,gBAAgB,SAAyB,SAA0C;AAClG,QAAM,SAAS,QAAQ,QAAQ,IAAI,QAAQ;AAC3C,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAQ,SAAS,cAAc;AACrC,QAAM,cAAc,OAAO,YAAY;AACvC,SAAO,MAAM,KAAK,CAAC,SAAS,YAAY,SAAS,IAAI,CAAC;AACvD;AAYO,SAAS,oBACf,SACA,SAC4B;AAC5B,QAAM,YAAY,UAAU,OAAO;AACnC,MAAI,UAAU,UAAU;AACvB,WAAO,EAAE,OAAO,MAAM,QAAQ,SAAS,UAAU;AAAA,EAClD;AAEA,MAAI,gBAAgB,SAAS,OAAO,GAAG;AACtC,WAAO,EAAE,OAAO,MAAM,QAAQ,iBAAiB,UAAU;AAAA,EAC1D;AAEA,SAAO,EAAE,OAAO,OAAO,QAAQ,MAAM,UAAU;AAChD;;;ACnCO,SAAS,yBAAyB,MAAc,SAAmC;AACzF,QAAM,UAAU,SAAS,cAAc;AACvC,QAAM,QAAQ,SAAS,YAAY;AACnC,QAAM,OAAO,SAAS,WAAW;AAEjC,SAAO;AAAA;AAAA,YAEI,IAAI;AAAA;AAAA;AAAA;AAAA,8BAIc,OAAO,KAAK,IAAI,GAAG,OAAO;AAAA,iCACvB,KAAK,KAAK,IAAI,GAAG,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvD;","names":[]}

package/dist/index.d.cts

@@ -0,0 +1,82 @@
+type DetectionMethod = "ua-match" | "signature-agent" | "heuristic";
+type DetectionResult = {
+    detected: true;
+    method: DetectionMethod;
+} | {
+    detected: false;
+    method: null;
+};
+/**
+ * Minimal request interface. Works with NextRequest, Request, or any object
+ * that has a headers.get() method.
+ */
+interface MinimalRequest {
+    headers: {
+        get(name: string): string | null;
+    };
+}
+
+/**
+ * Detects AI agents from HTTP request headers.
+ *
+ * Three detection layers (checked in order):
+ * 1. Known UA patterns (definitive)
+ * 2. Signature-Agent header (definitive, RFC 9421)
+ * 3. Missing sec-fetch-mode heuristic (catches unknown bots)
+ *
+ * Optimizes for recall over precision: serving markdown to a non-AI bot
+ * is low-harm; missing an AI agent means a worse experience.
+ */
+declare function isAIAgent(request: MinimalRequest): DetectionResult;
+
+interface AcceptMarkdownOptions {
+    mediaTypes?: string[];
+}
+/**
+ * Check if the request prefers markdown via the Accept header.
+ */
+declare function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean;
+interface ShouldServeMarkdownResult {
+    serve: boolean;
+    reason: "agent" | "accept-header" | null;
+    detection: DetectionResult;
+}
+/**
+ * Combines agent detection and content negotiation into one call.
+ * Returns whether to serve markdown and why.
+ */
+declare function shouldServeMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): ShouldServeMarkdownResult;
+
+interface NotFoundOptions {
+    sitemapUrl?: string;
+    indexUrl?: string;
+    baseUrl?: string;
+}
+/**
+ * Generates a markdown body for missing pages with links to discovery endpoints.
+ * Return with a 200 status (agents discard 404 response bodies).
+ */
+declare function generateNotFoundMarkdown(path: string, options?: NotFoundOptions): string;
+
+/**
+ * Layer 1: Known AI agent UA substrings (lowercase).
+ * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.
+ * Last reviewed: 2026-03-20
+ */
+declare const AI_AGENT_UA_PATTERNS: readonly string[];
+/**
+ * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).
+ */
+declare const SIGNATURE_AGENT_DOMAINS: readonly string[];
+/**
+ * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the
+ * heuristic layer (search engine crawlers, social previews, monitoring tools).
+ */
+declare const TRADITIONAL_BOT_PATTERNS: readonly string[];
+/**
+ * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).
+ * No word boundaries — keywords commonly appear in compound names.
+ */
+declare const BOT_LIKE_REGEX: RegExp;
+
+export { AI_AGENT_UA_PATTERNS, type AcceptMarkdownOptions, BOT_LIKE_REGEX, type DetectionMethod, type DetectionResult, type MinimalRequest, type NotFoundOptions, SIGNATURE_AGENT_DOMAINS, type ShouldServeMarkdownResult, TRADITIONAL_BOT_PATTERNS, acceptsMarkdown, generateNotFoundMarkdown, isAIAgent, shouldServeMarkdown };

package/dist/index.d.ts

@@ -0,0 +1,82 @@
+type DetectionMethod = "ua-match" | "signature-agent" | "heuristic";
+type DetectionResult = {
+    detected: true;
+    method: DetectionMethod;
+} | {
+    detected: false;
+    method: null;
+};
+/**
+ * Minimal request interface. Works with NextRequest, Request, or any object
+ * that has a headers.get() method.
+ */
+interface MinimalRequest {
+    headers: {
+        get(name: string): string | null;
+    };
+}
+
+/**
+ * Detects AI agents from HTTP request headers.
+ *
+ * Three detection layers (checked in order):
+ * 1. Known UA patterns (definitive)
+ * 2. Signature-Agent header (definitive, RFC 9421)
+ * 3. Missing sec-fetch-mode heuristic (catches unknown bots)
+ *
+ * Optimizes for recall over precision: serving markdown to a non-AI bot
+ * is low-harm; missing an AI agent means a worse experience.
+ */
+declare function isAIAgent(request: MinimalRequest): DetectionResult;
+
+interface AcceptMarkdownOptions {
+    mediaTypes?: string[];
+}
+/**
+ * Check if the request prefers markdown via the Accept header.
+ */
+declare function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean;
+interface ShouldServeMarkdownResult {
+    serve: boolean;
+    reason: "agent" | "accept-header" | null;
+    detection: DetectionResult;
+}
+/**
+ * Combines agent detection and content negotiation into one call.
+ * Returns whether to serve markdown and why.
+ */
+declare function shouldServeMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): ShouldServeMarkdownResult;
+
+interface NotFoundOptions {
+    sitemapUrl?: string;
+    indexUrl?: string;
+    baseUrl?: string;
+}
+/**
+ * Generates a markdown body for missing pages with links to discovery endpoints.
+ * Return with a 200 status (agents discard 404 response bodies).
+ */
+declare function generateNotFoundMarkdown(path: string, options?: NotFoundOptions): string;
+
+/**
+ * Layer 1: Known AI agent UA substrings (lowercase).
+ * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.
+ * Last reviewed: 2026-03-20
+ */
+declare const AI_AGENT_UA_PATTERNS: readonly string[];
+/**
+ * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).
+ */
+declare const SIGNATURE_AGENT_DOMAINS: readonly string[];
+/**
+ * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the
+ * heuristic layer (search engine crawlers, social previews, monitoring tools).
+ */
+declare const TRADITIONAL_BOT_PATTERNS: readonly string[];
+/**
+ * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).
+ * No word boundaries — keywords commonly appear in compound names.
+ */
+declare const BOT_LIKE_REGEX: RegExp;
+
+export { AI_AGENT_UA_PATTERNS, type AcceptMarkdownOptions, BOT_LIKE_REGEX, type DetectionMethod, type DetectionResult, type MinimalRequest, type NotFoundOptions, SIGNATURE_AGENT_DOMAINS, type ShouldServeMarkdownResult, TRADITIONAL_BOT_PATTERNS, acceptsMarkdown, generateNotFoundMarkdown, isAIAgent, shouldServeMarkdown };

package/dist/index.js

@@ -0,0 +1,141 @@
+// src/patterns.ts
+var AI_AGENT_UA_PATTERNS = [
+  // Anthropic — https://support.claude.com/en/articles/8896518
+  "claudebot",
+  "claude-searchbot",
+  "claude-user",
+  "anthropic-ai",
+  "claude-web",
+  // OpenAI — https://platform.openai.com/docs/bots
+  "chatgpt",
+  "gptbot",
+  "oai-searchbot",
+  "openai",
+  // Google AI
+  "gemini",
+  "bard",
+  "google-cloudvertexbot",
+  "google-extended",
+  // Meta
+  "meta-externalagent",
+  "meta-externalfetcher",
+  "meta-webindexer",
+  // Search/Research AI
+  "perplexity",
+  "youbot",
+  "you.com",
+  "deepseekbot",
+  // Coding assistants
+  "cursor",
+  "github-copilot",
+  "codeium",
+  "tabnine",
+  "sourcegraph",
+  // Other AI agents / data scrapers
+  "cohere-ai",
+  "bytespider",
+  "amazonbot",
+  "ai2bot",
+  "diffbot",
+  "omgili",
+  "omgilibot"
+];
+var SIGNATURE_AGENT_DOMAINS = ["chatgpt.com"];
+var TRADITIONAL_BOT_PATTERNS = [
+  "googlebot",
+  "bingbot",
+  "yandexbot",
+  "baiduspider",
+  "duckduckbot",
+  "slurp",
+  "msnbot",
+  "facebot",
+  "twitterbot",
+  "linkedinbot",
+  "whatsapp",
+  "telegrambot",
+  "pingdom",
+  "uptimerobot",
+  "newrelic",
+  "datadog",
+  "statuspage",
+  "site24x7",
+  "applebot"
+];
+var BOT_LIKE_REGEX = /bot|agent|fetch|crawl|spider|search/i;
+
+// src/detection.ts
+function isAIAgent(request) {
+  const userAgent = request.headers.get("user-agent");
+  const lowerUA = userAgent?.toLowerCase() ?? "";
+  if (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {
+    return { detected: true, method: "ua-match" };
+  }
+  const signatureAgent = request.headers.get("signature-agent");
+  if (signatureAgent) {
+    const lowerSig = signatureAgent.toLowerCase();
+    if (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {
+      return { detected: true, method: "signature-agent" };
+    }
+  }
+  const secFetchMode = request.headers.get("sec-fetch-mode");
+  if (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {
+    const isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));
+    if (!isTraditionalBot) {
+      return { detected: true, method: "heuristic" };
+    }
+  }
+  return { detected: false, method: null };
+}
+
+// src/negotiation.ts
+var DEFAULT_MARKDOWN_TYPES = ["text/markdown", "text/x-markdown"];
+function acceptsMarkdown(request, options) {
+  const accept = request.headers.get("accept");
+  if (!accept) return false;
+  const types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;
+  const lowerAccept = accept.toLowerCase();
+  return types.some((type) => lowerAccept.includes(type));
+}
+function shouldServeMarkdown(request, options) {
+  const detection = isAIAgent(request);
+  if (detection.detected) {
+    return { serve: true, reason: "agent", detection };
+  }
+  if (acceptsMarkdown(request, options)) {
+    return { serve: true, reason: "accept-header", detection };
+  }
+  return { serve: false, reason: null, detection };
+}
+
+// src/not-found.ts
+function generateNotFoundMarkdown(path, options) {
+  const sitemap = options?.sitemapUrl ?? "/sitemap.md";
+  const index = options?.indexUrl ?? "/llms.txt";
+  const base = options?.baseUrl ?? "";
+  return `# Page Not Found
+
+The URL \`${path}\` does not exist.
+
+## How to find the correct page
+
+1. **Browse the sitemap**: [${sitemap}](${base}${sitemap}) - A structured index of all pages
+2. **Browse the full index**: [${index}](${base}${index}) - Complete documentation index
+
+## Tips for requesting documentation
+
+- For markdown responses, append \`.md\` to URLs (e.g., \`/docs/functions.md\`)
+- Use \`Accept: text/markdown\` header for content negotiation
+`;
+}
+export {
+  AI_AGENT_UA_PATTERNS,
+  BOT_LIKE_REGEX,
+  SIGNATURE_AGENT_DOMAINS,
+  TRADITIONAL_BOT_PATTERNS,
+  acceptsMarkdown,
+  generateNotFoundMarkdown,
+  isAIAgent,
+  shouldServeMarkdown
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/patterns.ts","../src/detection.ts","../src/negotiation.ts","../src/not-found.ts"],"sourcesContent":["/**\n * Layer 1: Known AI agent UA substrings (lowercase).\n * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.\n * Last reviewed: 2026-03-20\n */\nexport const AI_AGENT_UA_PATTERNS: readonly string[] = [\n\t// Anthropic — https://support.claude.com/en/articles/8896518\n\t\"claudebot\",\n\t\"claude-searchbot\",\n\t\"claude-user\",\n\t\"anthropic-ai\",\n\t\"claude-web\",\n\n\t// OpenAI — https://platform.openai.com/docs/bots\n\t\"chatgpt\",\n\t\"gptbot\",\n\t\"oai-searchbot\",\n\t\"openai\",\n\n\t// Google AI\n\t\"gemini\",\n\t\"bard\",\n\t\"google-cloudvertexbot\",\n\t\"google-extended\",\n\n\t// Meta\n\t\"meta-externalagent\",\n\t\"meta-externalfetcher\",\n\t\"meta-webindexer\",\n\n\t// Search/Research AI\n\t\"perplexity\",\n\t\"youbot\",\n\t\"you.com\",\n\t\"deepseekbot\",\n\n\t// Coding assistants\n\t\"cursor\",\n\t\"github-copilot\",\n\t\"codeium\",\n\t\"tabnine\",\n\t\"sourcegraph\",\n\n\t// Other AI agents / data scrapers\n\t\"cohere-ai\",\n\t\"bytespider\",\n\t\"amazonbot\",\n\t\"ai2bot\",\n\t\"diffbot\",\n\t\"omgili\",\n\t\"omgilibot\",\n];\n\n/**\n * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).\n */\nexport const SIGNATURE_AGENT_DOMAINS: readonly string[] = [\"chatgpt.com\"];\n\n/**\n * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the\n * heuristic layer (search engine crawlers, social previews, monitoring tools).\n */\nexport const TRADITIONAL_BOT_PATTERNS: readonly string[] = [\n\t\"googlebot\",\n\t\"bingbot\",\n\t\"yandexbot\",\n\t\"baiduspider\",\n\t\"duckduckbot\",\n\t\"slurp\",\n\t\"msnbot\",\n\t\"facebot\",\n\t\"twitterbot\",\n\t\"linkedinbot\",\n\t\"whatsapp\",\n\t\"telegrambot\",\n\t\"pingdom\",\n\t\"uptimerobot\",\n\t\"newrelic\",\n\t\"datadog\",\n\t\"statuspage\",\n\t\"site24x7\",\n\t\"applebot\",\n];\n\n/**\n * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).\n * No word boundaries — keywords commonly appear in compound names.\n */\nexport const BOT_LIKE_REGEX: RegExp = /bot|agent|fetch|crawl|spider|search/i;\n","import {\n\tAI_AGENT_UA_PATTERNS,\n\tBOT_LIKE_REGEX,\n\tSIGNATURE_AGENT_DOMAINS,\n\tTRADITIONAL_BOT_PATTERNS,\n} from \"./patterns\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\n/**\n * Detects AI agents from HTTP request headers.\n *\n * Three detection layers (checked in order):\n * 1. Known UA patterns (definitive)\n * 2. Signature-Agent header (definitive, RFC 9421)\n * 3. Missing sec-fetch-mode heuristic (catches unknown bots)\n *\n * Optimizes for recall over precision: serving markdown to a non-AI bot\n * is low-harm; missing an AI agent means a worse experience.\n */\nexport function isAIAgent(request: MinimalRequest): DetectionResult {\n\tconst userAgent = request.headers.get(\"user-agent\");\n\tconst lowerUA = userAgent?.toLowerCase() ?? \"\";\n\n\t// Layer 1: Known UA pattern match\n\tif (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {\n\t\treturn { detected: true, method: \"ua-match\" };\n\t}\n\n\t// Layer 2: Signature-Agent header (RFC 9421, used by ChatGPT agent)\n\tconst signatureAgent = request.headers.get(\"signature-agent\");\n\tif (signatureAgent) {\n\t\tconst lowerSig = signatureAgent.toLowerCase();\n\t\tif (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {\n\t\t\treturn { detected: true, method: \"signature-agent\" };\n\t\t}\n\t}\n\n\t// Layer 3: Missing browser fingerprint heuristic\n\t// Real browsers (Chrome 76+, Firefox 90+, Safari 16.4+) send sec-fetch-mode\n\t// on navigation requests. Its absence signals a programmatic client.\n\tconst secFetchMode = request.headers.get(\"sec-fetch-mode\");\n\tif (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {\n\t\tconst isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));\n\t\tif (!isTraditionalBot) {\n\t\t\treturn { detected: true, method: \"heuristic\" };\n\t\t}\n\t}\n\n\treturn { detected: false, method: null };\n}\n","import { isAIAgent } from \"./detection\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\nconst DEFAULT_MARKDOWN_TYPES = [\"text/markdown\", \"text/x-markdown\"];\n\nexport interface AcceptMarkdownOptions {\n\tmediaTypes?: string[];\n}\n\n/**\n * Check if the request prefers markdown via the Accept header.\n */\nexport function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean {\n\tconst accept = request.headers.get(\"accept\");\n\tif (!accept) return false;\n\n\tconst types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;\n\tconst lowerAccept = accept.toLowerCase();\n\treturn types.some((type) => lowerAccept.includes(type));\n}\n\nexport interface ShouldServeMarkdownResult {\n\tserve: boolean;\n\treason: \"agent\" | \"accept-header\" | null;\n\tdetection: DetectionResult;\n}\n\n/**\n * Combines agent detection and content negotiation into one call.\n * Returns whether to serve markdown and why.\n */\nexport function shouldServeMarkdown(\n\trequest: MinimalRequest,\n\toptions?: AcceptMarkdownOptions,\n): ShouldServeMarkdownResult {\n\tconst detection = isAIAgent(request);\n\tif (detection.detected) {\n\t\treturn { serve: true, reason: \"agent\", detection };\n\t}\n\n\tif (acceptsMarkdown(request, options)) {\n\t\treturn { serve: true, reason: \"accept-header\", detection };\n\t}\n\n\treturn { serve: false, reason: null, detection };\n}\n","export interface NotFoundOptions {\n\tsitemapUrl?: string;\n\tindexUrl?: string;\n\tbaseUrl?: string;\n}\n\n/**\n * Generates a markdown body for missing pages with links to discovery endpoints.\n * Return with a 200 status (agents discard 404 response bodies).\n */\nexport function generateNotFoundMarkdown(path: string, options?: NotFoundOptions): string {\n\tconst sitemap = options?.sitemapUrl ?? \"/sitemap.md\";\n\tconst index = options?.indexUrl ?? \"/llms.txt\";\n\tconst base = options?.baseUrl ?? \"\";\n\n\treturn `# Page Not Found\n\nThe URL \\`${path}\\` does not exist.\n\n## How to find the correct page\n\n1. **Browse the sitemap**: [${sitemap}](${base}${sitemap}) - A structured index of all pages\n2. **Browse the full index**: [${index}](${base}${index}) - Complete documentation index\n\n## Tips for requesting documentation\n\n- For markdown responses, append \\`.md\\` to URLs (e.g., \\`/docs/functions.md\\`)\n- Use \\`Accept: text/markdown\\` header for content negotiation\n`;\n}\n"],"mappings":";AAKO,IAAM,uBAA0C;AAAA;AAAA,EAEtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAKO,IAAM,0BAA6C,CAAC,aAAa;AAMjE,IAAM,2BAA8C;AAAA,EAC1D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAMO,IAAM,iBAAyB;;;ACrE/B,SAAS,UAAU,SAA0C;AACnE,QAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY;AAClD,QAAM,UAAU,WAAW,YAAY,KAAK;AAG5C,MAAI,WAAW,qBAAqB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC,GAAG;AACjF,WAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAAA,EAC7C;AAGA,QAAM,iBAAiB,QAAQ,QAAQ,IAAI,iBAAiB;AAC5D,MAAI,gBAAgB;AACnB,UAAM,WAAW,eAAe,YAAY;AAC5C,QAAI,wBAAwB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC,GAAG;AACxE,aAAO,EAAE,UAAU,MAAM,QAAQ,kBAAkB;AAAA,IACpD;AAAA,EACD;AAKA,QAAM,eAAe,QAAQ,QAAQ,IAAI,gBAAgB;AACzD,MAAI,CAAC,gBAAgB,WAAW,eAAe,KAAK,OAAO,GAAG;AAC7D,UAAM,mBAAmB,yBAAyB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC;AAC7F,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,UAAU,MAAM,QAAQ,YAAY;AAAA,IAC9C;AAAA,EACD;AAEA,SAAO,EAAE,UAAU,OAAO,QAAQ,KAAK;AACxC;;;AC9CA,IAAM,yBAAyB,CAAC,iBAAiB,iBAAiB;AAS3D,SAAS,gBAAgB,SAAyB,SAA0C;AAClG,QAAM,SAAS,QAAQ,QAAQ,IAAI,QAAQ;AAC3C,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAQ,SAAS,cAAc;AACrC,QAAM,cAAc,OAAO,YAAY;AACvC,SAAO,MAAM,KAAK,CAAC,SAAS,YAAY,SAAS,IAAI,CAAC;AACvD;AAYO,SAAS,oBACf,SACA,SAC4B;AAC5B,QAAM,YAAY,UAAU,OAAO;AACnC,MAAI,UAAU,UAAU;AACvB,WAAO,EAAE,OAAO,MAAM,QAAQ,SAAS,UAAU;AAAA,EAClD;AAEA,MAAI,gBAAgB,SAAS,OAAO,GAAG;AACtC,WAAO,EAAE,OAAO,MAAM,QAAQ,iBAAiB,UAAU;AAAA,EAC1D;AAEA,SAAO,EAAE,OAAO,OAAO,QAAQ,MAAM,UAAU;AAChD;;;ACnCO,SAAS,yBAAyB,MAAc,SAAmC;AACzF,QAAM,UAAU,SAAS,cAAc;AACvC,QAAM,QAAQ,SAAS,YAAY;AACnC,QAAM,OAAO,SAAS,WAAW;AAEjC,SAAO;AAAA;AAAA,YAEI,IAAI;AAAA;AAAA;AAAA;AAAA,8BAIc,OAAO,KAAK,IAAI,GAAG,OAAO;AAAA,iCACvB,KAAK,KAAK,IAAI,GAAG,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvD;","names":[]}