@vercel/agent-readability 0.1.0

package/dist/next/index.cjs

@@ -0,0 +1,166 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/next/index.ts
+var next_exports = {};
+__export(next_exports, {
+  agentReadabilityMatcher: () => agentReadabilityMatcher,
+  withAgentReadability: () => withAgentReadability
+});
+module.exports = __toCommonJS(next_exports);
+var import_server = require("next/server");
+
+// src/patterns.ts
+var AI_AGENT_UA_PATTERNS = [
+  // Anthropic — https://support.claude.com/en/articles/8896518
+  "claudebot",
+  "claude-searchbot",
+  "claude-user",
+  "anthropic-ai",
+  "claude-web",
+  // OpenAI — https://platform.openai.com/docs/bots
+  "chatgpt",
+  "gptbot",
+  "oai-searchbot",
+  "openai",
+  // Google AI
+  "gemini",
+  "bard",
+  "google-cloudvertexbot",
+  "google-extended",
+  // Meta
+  "meta-externalagent",
+  "meta-externalfetcher",
+  "meta-webindexer",
+  // Search/Research AI
+  "perplexity",
+  "youbot",
+  "you.com",
+  "deepseekbot",
+  // Coding assistants
+  "cursor",
+  "github-copilot",
+  "codeium",
+  "tabnine",
+  "sourcegraph",
+  // Other AI agents / data scrapers
+  "cohere-ai",
+  "bytespider",
+  "amazonbot",
+  "ai2bot",
+  "diffbot",
+  "omgili",
+  "omgilibot"
+];
+var SIGNATURE_AGENT_DOMAINS = ["chatgpt.com"];
+var TRADITIONAL_BOT_PATTERNS = [
+  "googlebot",
+  "bingbot",
+  "yandexbot",
+  "baiduspider",
+  "duckduckbot",
+  "slurp",
+  "msnbot",
+  "facebot",
+  "twitterbot",
+  "linkedinbot",
+  "whatsapp",
+  "telegrambot",
+  "pingdom",
+  "uptimerobot",
+  "newrelic",
+  "datadog",
+  "statuspage",
+  "site24x7",
+  "applebot"
+];
+var BOT_LIKE_REGEX = /bot|agent|fetch|crawl|spider|search/i;
+
+// src/detection.ts
+function isAIAgent(request) {
+  const userAgent = request.headers.get("user-agent");
+  const lowerUA = userAgent?.toLowerCase() ?? "";
+  if (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {
+    return { detected: true, method: "ua-match" };
+  }
+  const signatureAgent = request.headers.get("signature-agent");
+  if (signatureAgent) {
+    const lowerSig = signatureAgent.toLowerCase();
+    if (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {
+      return { detected: true, method: "signature-agent" };
+    }
+  }
+  const secFetchMode = request.headers.get("sec-fetch-mode");
+  if (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {
+    const isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));
+    if (!isTraditionalBot) {
+      return { detected: true, method: "heuristic" };
+    }
+  }
+  return { detected: false, method: null };
+}
+
+// src/negotiation.ts
+var DEFAULT_MARKDOWN_TYPES = ["text/markdown", "text/x-markdown"];
+function acceptsMarkdown(request, options) {
+  const accept = request.headers.get("accept");
+  if (!accept) return false;
+  const types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;
+  const lowerAccept = accept.toLowerCase();
+  return types.some((type) => lowerAccept.includes(type));
+}
+
+// src/next/index.ts
+function withAgentReadability(options, handler) {
+  return async (req, event) => {
+    const { pathname } = req.nextUrl;
+    const prefix = options.docsPrefix ?? "/docs";
+    if (pathname.startsWith(prefix)) {
+      const result = isAIAgent(req);
+      const markdownAccepted = acceptsMarkdown(req);
+      if (result.detected || markdownAccepted) {
+        if (options.onDetection) {
+          const method = result.detected ? result.method : "accept-header";
+          const detection = options.onDetection({
+            path: pathname,
+            method,
+            userAgent: req.headers.get("user-agent")
+          });
+          if (detection instanceof Promise) {
+            event.waitUntil(detection);
+          }
+        }
+        const rewriteUrl = req.nextUrl.clone();
+        rewriteUrl.pathname = options.rewrite(pathname);
+        return import_server.NextResponse.rewrite(rewriteUrl);
+      }
+    }
+    if (handler) return handler(req, event);
+    return import_server.NextResponse.next();
+  };
+}
+var agentReadabilityMatcher = [
+  "/((?!_next|api|.*\\..*|favicon|manifest|robots|health|status).*)"
+];
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  agentReadabilityMatcher,
+  withAgentReadability
+});
+//# sourceMappingURL=index.cjs.map

package/dist/next/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/next/index.ts","../../src/patterns.ts","../../src/detection.ts","../../src/negotiation.ts"],"sourcesContent":["import type { NextFetchEvent, NextRequest } from \"next/server\";\nimport { NextResponse } from \"next/server\";\nimport { isAIAgent } from \"../detection\";\nimport { acceptsMarkdown } from \"../negotiation\";\nimport type { DetectionMethod } from \"../types\";\n\nexport interface AgentReadabilityOptions {\n\t/** URL prefix to intercept. Default: '/docs' */\n\tdocsPrefix?: string;\n\t/** Maps request path to markdown route */\n\trewrite: (pathname: string) => string;\n\t/** Fire-and-forget callback. Async returns are passed to event.waitUntil(). */\n\tonDetection?: (info: {\n\t\tpath: string;\n\t\tmethod: DetectionMethod | \"accept-header\";\n\t\tuserAgent: string | null;\n\t}) => void | Promise<void>;\n}\n\nexport type MiddlewareHandler = (\n\treq: NextRequest,\n\tevent: NextFetchEvent,\n) => NextResponse | Response | Promise<NextResponse | Response>;\n\n/**\n * Higher-order function that wraps Next.js middleware with AI agent detection.\n * Detected AI agents or requests with Accept: text/markdown are rewritten\n * to the markdown route returned by `options.rewrite()`.\n *\n * Composes with existing middleware:\n * ```ts\n * export default withAgentReadability(\n *   { rewrite: (p) => `/md${p}` },\n *   (req, event) => existingMiddleware(req, event),\n * )\n * ```\n */\nexport function withAgentReadability(\n\toptions: AgentReadabilityOptions,\n\thandler?: MiddlewareHandler,\n): MiddlewareHandler {\n\treturn async (req: NextRequest, event: NextFetchEvent) => {\n\t\tconst { pathname } = req.nextUrl;\n\t\tconst prefix = options.docsPrefix ?? \"/docs\";\n\n\t\tif (pathname.startsWith(prefix)) {\n\t\t\tconst result = isAIAgent(req);\n\t\t\tconst markdownAccepted = acceptsMarkdown(req);\n\n\t\t\tif (result.detected || markdownAccepted) {\n\t\t\t\tif (options.onDetection) {\n\t\t\t\t\tconst method = result.detected ? result.method : \"accept-header\";\n\t\t\t\t\tconst detection = options.onDetection({\n\t\t\t\t\t\tpath: pathname,\n\t\t\t\t\t\tmethod,\n\t\t\t\t\t\tuserAgent: req.headers.get(\"user-agent\"),\n\t\t\t\t\t});\n\t\t\t\t\tif (detection instanceof Promise) {\n\t\t\t\t\t\tevent.waitUntil(detection);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tconst rewriteUrl = req.nextUrl.clone();\n\t\t\t\trewriteUrl.pathname = options.rewrite(pathname);\n\t\t\t\treturn NextResponse.rewrite(rewriteUrl);\n\t\t\t}\n\t\t}\n\n\t\tif (handler) return handler(req, event);\n\t\treturn NextResponse.next();\n\t};\n}\n\n/** Recommended matcher that excludes Next.js internals and static files. */\nexport const agentReadabilityMatcher = [\n\t\"/((?!_next|api|.*\\\\..*|favicon|manifest|robots|health|status).*)\",\n] as const;\n","/**\n * Layer 1: Known AI agent UA substrings (lowercase).\n * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.\n * Last reviewed: 2026-03-20\n */\nexport const AI_AGENT_UA_PATTERNS: readonly string[] = [\n\t// Anthropic — https://support.claude.com/en/articles/8896518\n\t\"claudebot\",\n\t\"claude-searchbot\",\n\t\"claude-user\",\n\t\"anthropic-ai\",\n\t\"claude-web\",\n\n\t// OpenAI — https://platform.openai.com/docs/bots\n\t\"chatgpt\",\n\t\"gptbot\",\n\t\"oai-searchbot\",\n\t\"openai\",\n\n\t// Google AI\n\t\"gemini\",\n\t\"bard\",\n\t\"google-cloudvertexbot\",\n\t\"google-extended\",\n\n\t// Meta\n\t\"meta-externalagent\",\n\t\"meta-externalfetcher\",\n\t\"meta-webindexer\",\n\n\t// Search/Research AI\n\t\"perplexity\",\n\t\"youbot\",\n\t\"you.com\",\n\t\"deepseekbot\",\n\n\t// Coding assistants\n\t\"cursor\",\n\t\"github-copilot\",\n\t\"codeium\",\n\t\"tabnine\",\n\t\"sourcegraph\",\n\n\t// Other AI agents / data scrapers\n\t\"cohere-ai\",\n\t\"bytespider\",\n\t\"amazonbot\",\n\t\"ai2bot\",\n\t\"diffbot\",\n\t\"omgili\",\n\t\"omgilibot\",\n];\n\n/**\n * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).\n */\nexport const SIGNATURE_AGENT_DOMAINS: readonly string[] = [\"chatgpt.com\"];\n\n/**\n * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the\n * heuristic layer (search engine crawlers, social previews, monitoring tools).\n */\nexport const TRADITIONAL_BOT_PATTERNS: readonly string[] = [\n\t\"googlebot\",\n\t\"bingbot\",\n\t\"yandexbot\",\n\t\"baiduspider\",\n\t\"duckduckbot\",\n\t\"slurp\",\n\t\"msnbot\",\n\t\"facebot\",\n\t\"twitterbot\",\n\t\"linkedinbot\",\n\t\"whatsapp\",\n\t\"telegrambot\",\n\t\"pingdom\",\n\t\"uptimerobot\",\n\t\"newrelic\",\n\t\"datadog\",\n\t\"statuspage\",\n\t\"site24x7\",\n\t\"applebot\",\n];\n\n/**\n * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).\n * No word boundaries — keywords commonly appear in compound names.\n */\nexport const BOT_LIKE_REGEX: RegExp = /bot|agent|fetch|crawl|spider|search/i;\n","import {\n\tAI_AGENT_UA_PATTERNS,\n\tBOT_LIKE_REGEX,\n\tSIGNATURE_AGENT_DOMAINS,\n\tTRADITIONAL_BOT_PATTERNS,\n} from \"./patterns\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\n/**\n * Detects AI agents from HTTP request headers.\n *\n * Three detection layers (checked in order):\n * 1. Known UA patterns (definitive)\n * 2. Signature-Agent header (definitive, RFC 9421)\n * 3. Missing sec-fetch-mode heuristic (catches unknown bots)\n *\n * Optimizes for recall over precision: serving markdown to a non-AI bot\n * is low-harm; missing an AI agent means a worse experience.\n */\nexport function isAIAgent(request: MinimalRequest): DetectionResult {\n\tconst userAgent = request.headers.get(\"user-agent\");\n\tconst lowerUA = userAgent?.toLowerCase() ?? \"\";\n\n\t// Layer 1: Known UA pattern match\n\tif (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {\n\t\treturn { detected: true, method: \"ua-match\" };\n\t}\n\n\t// Layer 2: Signature-Agent header (RFC 9421, used by ChatGPT agent)\n\tconst signatureAgent = request.headers.get(\"signature-agent\");\n\tif (signatureAgent) {\n\t\tconst lowerSig = signatureAgent.toLowerCase();\n\t\tif (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {\n\t\t\treturn { detected: true, method: \"signature-agent\" };\n\t\t}\n\t}\n\n\t// Layer 3: Missing browser fingerprint heuristic\n\t// Real browsers (Chrome 76+, Firefox 90+, Safari 16.4+) send sec-fetch-mode\n\t// on navigation requests. Its absence signals a programmatic client.\n\tconst secFetchMode = request.headers.get(\"sec-fetch-mode\");\n\tif (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {\n\t\tconst isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));\n\t\tif (!isTraditionalBot) {\n\t\t\treturn { detected: true, method: \"heuristic\" };\n\t\t}\n\t}\n\n\treturn { detected: false, method: null };\n}\n","import { isAIAgent } from \"./detection\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\nconst DEFAULT_MARKDOWN_TYPES = [\"text/markdown\", \"text/x-markdown\"];\n\nexport interface AcceptMarkdownOptions {\n\tmediaTypes?: string[];\n}\n\n/**\n * Check if the request prefers markdown via the Accept header.\n */\nexport function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean {\n\tconst accept = request.headers.get(\"accept\");\n\tif (!accept) return false;\n\n\tconst types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;\n\tconst lowerAccept = accept.toLowerCase();\n\treturn types.some((type) => lowerAccept.includes(type));\n}\n\nexport interface ShouldServeMarkdownResult {\n\tserve: boolean;\n\treason: \"agent\" | \"accept-header\" | null;\n\tdetection: DetectionResult;\n}\n\n/**\n * Combines agent detection and content negotiation into one call.\n * Returns whether to serve markdown and why.\n */\nexport function shouldServeMarkdown(\n\trequest: MinimalRequest,\n\toptions?: AcceptMarkdownOptions,\n): ShouldServeMarkdownResult {\n\tconst detection = isAIAgent(request);\n\tif (detection.detected) {\n\t\treturn { serve: true, reason: \"agent\", detection };\n\t}\n\n\tif (acceptsMarkdown(request, options)) {\n\t\treturn { serve: true, reason: \"accept-header\", detection };\n\t}\n\n\treturn { serve: false, reason: null, detection };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,oBAA6B;;;ACItB,IAAM,uBAA0C;AAAA;AAAA,EAEtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAKO,IAAM,0BAA6C,CAAC,aAAa;AAMjE,IAAM,2BAA8C;AAAA,EAC1D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAMO,IAAM,iBAAyB;;;ACrE/B,SAAS,UAAU,SAA0C;AACnE,QAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY;AAClD,QAAM,UAAU,WAAW,YAAY,KAAK;AAG5C,MAAI,WAAW,qBAAqB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC,GAAG;AACjF,WAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAAA,EAC7C;AAGA,QAAM,iBAAiB,QAAQ,QAAQ,IAAI,iBAAiB;AAC5D,MAAI,gBAAgB;AACnB,UAAM,WAAW,eAAe,YAAY;AAC5C,QAAI,wBAAwB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC,GAAG;AACxE,aAAO,EAAE,UAAU,MAAM,QAAQ,kBAAkB;AAAA,IACpD;AAAA,EACD;AAKA,QAAM,eAAe,QAAQ,QAAQ,IAAI,gBAAgB;AACzD,MAAI,CAAC,gBAAgB,WAAW,eAAe,KAAK,OAAO,GAAG;AAC7D,UAAM,mBAAmB,yBAAyB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC;AAC7F,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,UAAU,MAAM,QAAQ,YAAY;AAAA,IAC9C;AAAA,EACD;AAEA,SAAO,EAAE,UAAU,OAAO,QAAQ,KAAK;AACxC;;;AC9CA,IAAM,yBAAyB,CAAC,iBAAiB,iBAAiB;AAS3D,SAAS,gBAAgB,SAAyB,SAA0C;AAClG,QAAM,SAAS,QAAQ,QAAQ,IAAI,QAAQ;AAC3C,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAQ,SAAS,cAAc;AACrC,QAAM,cAAc,OAAO,YAAY;AACvC,SAAO,MAAM,KAAK,CAAC,SAAS,YAAY,SAAS,IAAI,CAAC;AACvD;;;AHkBO,SAAS,qBACf,SACA,SACoB;AACpB,SAAO,OAAO,KAAkB,UAA0B;AACzD,UAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAM,SAAS,QAAQ,cAAc;AAErC,QAAI,SAAS,WAAW,MAAM,GAAG;AAChC,YAAM,SAAS,UAAU,GAAG;AAC5B,YAAM,mBAAmB,gBAAgB,GAAG;AAE5C,UAAI,OAAO,YAAY,kBAAkB;AACxC,YAAI,QAAQ,aAAa;AACxB,gBAAM,SAAS,OAAO,WAAW,OAAO,SAAS;AACjD,gBAAM,YAAY,QAAQ,YAAY;AAAA,YACrC,MAAM;AAAA,YACN;AAAA,YACA,WAAW,IAAI,QAAQ,IAAI,YAAY;AAAA,UACxC,CAAC;AACD,cAAI,qBAAqB,SAAS;AACjC,kBAAM,UAAU,SAAS;AAAA,UAC1B;AAAA,QACD;AAEA,cAAM,aAAa,IAAI,QAAQ,MAAM;AACrC,mBAAW,WAAW,QAAQ,QAAQ,QAAQ;AAC9C,eAAO,2BAAa,QAAQ,UAAU;AAAA,MACvC;AAAA,IACD;AAEA,QAAI,QAAS,QAAO,QAAQ,KAAK,KAAK;AACtC,WAAO,2BAAa,KAAK;AAAA,EAC1B;AACD;AAGO,IAAM,0BAA0B;AAAA,EACtC;AACD;","names":[]}

package/dist/next/index.d.cts

@@ -0,0 +1,35 @@
+import { NextRequest, NextFetchEvent, NextResponse } from 'next/server';
+
+type DetectionMethod = "ua-match" | "signature-agent" | "heuristic";
+
+interface AgentReadabilityOptions {
+    /** URL prefix to intercept. Default: '/docs' */
+    docsPrefix?: string;
+    /** Maps request path to markdown route */
+    rewrite: (pathname: string) => string;
+    /** Fire-and-forget callback. Async returns are passed to event.waitUntil(). */
+    onDetection?: (info: {
+        path: string;
+        method: DetectionMethod | "accept-header";
+        userAgent: string | null;
+    }) => void | Promise<void>;
+}
+type MiddlewareHandler = (req: NextRequest, event: NextFetchEvent) => NextResponse | Response | Promise<NextResponse | Response>;
+/**
+ * Higher-order function that wraps Next.js middleware with AI agent detection.
+ * Detected AI agents or requests with Accept: text/markdown are rewritten
+ * to the markdown route returned by `options.rewrite()`.
+ *
+ * Composes with existing middleware:
+ * ```ts
+ * export default withAgentReadability(
+ *   { rewrite: (p) => `/md${p}` },
+ *   (req, event) => existingMiddleware(req, event),
+ * )
+ * ```
+ */
+declare function withAgentReadability(options: AgentReadabilityOptions, handler?: MiddlewareHandler): MiddlewareHandler;
+/** Recommended matcher that excludes Next.js internals and static files. */
+declare const agentReadabilityMatcher: readonly ["/((?!_next|api|.*\\..*|favicon|manifest|robots|health|status).*)"];
+
+export { type AgentReadabilityOptions, type MiddlewareHandler, agentReadabilityMatcher, withAgentReadability };

package/dist/next/index.d.ts

@@ -0,0 +1,35 @@
+import { NextRequest, NextFetchEvent, NextResponse } from 'next/server';
+
+type DetectionMethod = "ua-match" | "signature-agent" | "heuristic";
+
+interface AgentReadabilityOptions {
+    /** URL prefix to intercept. Default: '/docs' */
+    docsPrefix?: string;
+    /** Maps request path to markdown route */
+    rewrite: (pathname: string) => string;
+    /** Fire-and-forget callback. Async returns are passed to event.waitUntil(). */
+    onDetection?: (info: {
+        path: string;
+        method: DetectionMethod | "accept-header";
+        userAgent: string | null;
+    }) => void | Promise<void>;
+}
+type MiddlewareHandler = (req: NextRequest, event: NextFetchEvent) => NextResponse | Response | Promise<NextResponse | Response>;
+/**
+ * Higher-order function that wraps Next.js middleware with AI agent detection.
+ * Detected AI agents or requests with Accept: text/markdown are rewritten
+ * to the markdown route returned by `options.rewrite()`.
+ *
+ * Composes with existing middleware:
+ * ```ts
+ * export default withAgentReadability(
+ *   { rewrite: (p) => `/md${p}` },
+ *   (req, event) => existingMiddleware(req, event),
+ * )
+ * ```
+ */
+declare function withAgentReadability(options: AgentReadabilityOptions, handler?: MiddlewareHandler): MiddlewareHandler;
+/** Recommended matcher that excludes Next.js internals and static files. */
+declare const agentReadabilityMatcher: readonly ["/((?!_next|api|.*\\..*|favicon|manifest|robots|health|status).*)"];
+
+export { type AgentReadabilityOptions, type MiddlewareHandler, agentReadabilityMatcher, withAgentReadability };

package/dist/next/index.js

@@ -0,0 +1,140 @@
+// src/next/index.ts
+import { NextResponse } from "next/server";
+
+// src/patterns.ts
+var AI_AGENT_UA_PATTERNS = [
+  // Anthropic — https://support.claude.com/en/articles/8896518
+  "claudebot",
+  "claude-searchbot",
+  "claude-user",
+  "anthropic-ai",
+  "claude-web",
+  // OpenAI — https://platform.openai.com/docs/bots
+  "chatgpt",
+  "gptbot",
+  "oai-searchbot",
+  "openai",
+  // Google AI
+  "gemini",
+  "bard",
+  "google-cloudvertexbot",
+  "google-extended",
+  // Meta
+  "meta-externalagent",
+  "meta-externalfetcher",
+  "meta-webindexer",
+  // Search/Research AI
+  "perplexity",
+  "youbot",
+  "you.com",
+  "deepseekbot",
+  // Coding assistants
+  "cursor",
+  "github-copilot",
+  "codeium",
+  "tabnine",
+  "sourcegraph",
+  // Other AI agents / data scrapers
+  "cohere-ai",
+  "bytespider",
+  "amazonbot",
+  "ai2bot",
+  "diffbot",
+  "omgili",
+  "omgilibot"
+];
+var SIGNATURE_AGENT_DOMAINS = ["chatgpt.com"];
+var TRADITIONAL_BOT_PATTERNS = [
+  "googlebot",
+  "bingbot",
+  "yandexbot",
+  "baiduspider",
+  "duckduckbot",
+  "slurp",
+  "msnbot",
+  "facebot",
+  "twitterbot",
+  "linkedinbot",
+  "whatsapp",
+  "telegrambot",
+  "pingdom",
+  "uptimerobot",
+  "newrelic",
+  "datadog",
+  "statuspage",
+  "site24x7",
+  "applebot"
+];
+var BOT_LIKE_REGEX = /bot|agent|fetch|crawl|spider|search/i;
+
+// src/detection.ts
+function isAIAgent(request) {
+  const userAgent = request.headers.get("user-agent");
+  const lowerUA = userAgent?.toLowerCase() ?? "";
+  if (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {
+    return { detected: true, method: "ua-match" };
+  }
+  const signatureAgent = request.headers.get("signature-agent");
+  if (signatureAgent) {
+    const lowerSig = signatureAgent.toLowerCase();
+    if (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {
+      return { detected: true, method: "signature-agent" };
+    }
+  }
+  const secFetchMode = request.headers.get("sec-fetch-mode");
+  if (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {
+    const isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));
+    if (!isTraditionalBot) {
+      return { detected: true, method: "heuristic" };
+    }
+  }
+  return { detected: false, method: null };
+}
+
+// src/negotiation.ts
+var DEFAULT_MARKDOWN_TYPES = ["text/markdown", "text/x-markdown"];
+function acceptsMarkdown(request, options) {
+  const accept = request.headers.get("accept");
+  if (!accept) return false;
+  const types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;
+  const lowerAccept = accept.toLowerCase();
+  return types.some((type) => lowerAccept.includes(type));
+}
+
+// src/next/index.ts
+function withAgentReadability(options, handler) {
+  return async (req, event) => {
+    const { pathname } = req.nextUrl;
+    const prefix = options.docsPrefix ?? "/docs";
+    if (pathname.startsWith(prefix)) {
+      const result = isAIAgent(req);
+      const markdownAccepted = acceptsMarkdown(req);
+      if (result.detected || markdownAccepted) {
+        if (options.onDetection) {
+          const method = result.detected ? result.method : "accept-header";
+          const detection = options.onDetection({
+            path: pathname,
+            method,
+            userAgent: req.headers.get("user-agent")
+          });
+          if (detection instanceof Promise) {
+            event.waitUntil(detection);
+          }
+        }
+        const rewriteUrl = req.nextUrl.clone();
+        rewriteUrl.pathname = options.rewrite(pathname);
+        return NextResponse.rewrite(rewriteUrl);
+      }
+    }
+    if (handler) return handler(req, event);
+    return NextResponse.next();
+  };
+}
+var agentReadabilityMatcher = [
+  "/((?!_next|api|.*\\..*|favicon|manifest|robots|health|status).*)"
+];
+export {
+  agentReadabilityMatcher,
+  withAgentReadability
+};
+//# sourceMappingURL=index.js.map

package/dist/next/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/next/index.ts","../../src/patterns.ts","../../src/detection.ts","../../src/negotiation.ts"],"sourcesContent":["import type { NextFetchEvent, NextRequest } from \"next/server\";\nimport { NextResponse } from \"next/server\";\nimport { isAIAgent } from \"../detection\";\nimport { acceptsMarkdown } from \"../negotiation\";\nimport type { DetectionMethod } from \"../types\";\n\nexport interface AgentReadabilityOptions {\n\t/** URL prefix to intercept. Default: '/docs' */\n\tdocsPrefix?: string;\n\t/** Maps request path to markdown route */\n\trewrite: (pathname: string) => string;\n\t/** Fire-and-forget callback. Async returns are passed to event.waitUntil(). */\n\tonDetection?: (info: {\n\t\tpath: string;\n\t\tmethod: DetectionMethod | \"accept-header\";\n\t\tuserAgent: string | null;\n\t}) => void | Promise<void>;\n}\n\nexport type MiddlewareHandler = (\n\treq: NextRequest,\n\tevent: NextFetchEvent,\n) => NextResponse | Response | Promise<NextResponse | Response>;\n\n/**\n * Higher-order function that wraps Next.js middleware with AI agent detection.\n * Detected AI agents or requests with Accept: text/markdown are rewritten\n * to the markdown route returned by `options.rewrite()`.\n *\n * Composes with existing middleware:\n * ```ts\n * export default withAgentReadability(\n *   { rewrite: (p) => `/md${p}` },\n *   (req, event) => existingMiddleware(req, event),\n * )\n * ```\n */\nexport function withAgentReadability(\n\toptions: AgentReadabilityOptions,\n\thandler?: MiddlewareHandler,\n): MiddlewareHandler {\n\treturn async (req: NextRequest, event: NextFetchEvent) => {\n\t\tconst { pathname } = req.nextUrl;\n\t\tconst prefix = options.docsPrefix ?? \"/docs\";\n\n\t\tif (pathname.startsWith(prefix)) {\n\t\t\tconst result = isAIAgent(req);\n\t\t\tconst markdownAccepted = acceptsMarkdown(req);\n\n\t\t\tif (result.detected || markdownAccepted) {\n\t\t\t\tif (options.onDetection) {\n\t\t\t\t\tconst method = result.detected ? result.method : \"accept-header\";\n\t\t\t\t\tconst detection = options.onDetection({\n\t\t\t\t\t\tpath: pathname,\n\t\t\t\t\t\tmethod,\n\t\t\t\t\t\tuserAgent: req.headers.get(\"user-agent\"),\n\t\t\t\t\t});\n\t\t\t\t\tif (detection instanceof Promise) {\n\t\t\t\t\t\tevent.waitUntil(detection);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tconst rewriteUrl = req.nextUrl.clone();\n\t\t\t\trewriteUrl.pathname = options.rewrite(pathname);\n\t\t\t\treturn NextResponse.rewrite(rewriteUrl);\n\t\t\t}\n\t\t}\n\n\t\tif (handler) return handler(req, event);\n\t\treturn NextResponse.next();\n\t};\n}\n\n/** Recommended matcher that excludes Next.js internals and static files. */\nexport const agentReadabilityMatcher = [\n\t\"/((?!_next|api|.*\\\\..*|favicon|manifest|robots|health|status).*)\",\n] as const;\n","/**\n * Layer 1: Known AI agent UA substrings (lowercase).\n * Curated from https://bots.fyi/?tags=ai_assistant + official vendor docs.\n * Last reviewed: 2026-03-20\n */\nexport const AI_AGENT_UA_PATTERNS: readonly string[] = [\n\t// Anthropic — https://support.claude.com/en/articles/8896518\n\t\"claudebot\",\n\t\"claude-searchbot\",\n\t\"claude-user\",\n\t\"anthropic-ai\",\n\t\"claude-web\",\n\n\t// OpenAI — https://platform.openai.com/docs/bots\n\t\"chatgpt\",\n\t\"gptbot\",\n\t\"oai-searchbot\",\n\t\"openai\",\n\n\t// Google AI\n\t\"gemini\",\n\t\"bard\",\n\t\"google-cloudvertexbot\",\n\t\"google-extended\",\n\n\t// Meta\n\t\"meta-externalagent\",\n\t\"meta-externalfetcher\",\n\t\"meta-webindexer\",\n\n\t// Search/Research AI\n\t\"perplexity\",\n\t\"youbot\",\n\t\"you.com\",\n\t\"deepseekbot\",\n\n\t// Coding assistants\n\t\"cursor\",\n\t\"github-copilot\",\n\t\"codeium\",\n\t\"tabnine\",\n\t\"sourcegraph\",\n\n\t// Other AI agents / data scrapers\n\t\"cohere-ai\",\n\t\"bytespider\",\n\t\"amazonbot\",\n\t\"ai2bot\",\n\t\"diffbot\",\n\t\"omgili\",\n\t\"omgilibot\",\n];\n\n/**\n * Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).\n */\nexport const SIGNATURE_AGENT_DOMAINS: readonly string[] = [\"chatgpt.com\"];\n\n/**\n * Layer 3: Traditional bot exclusion list. Bots that should NOT trigger the\n * heuristic layer (search engine crawlers, social previews, monitoring tools).\n */\nexport const TRADITIONAL_BOT_PATTERNS: readonly string[] = [\n\t\"googlebot\",\n\t\"bingbot\",\n\t\"yandexbot\",\n\t\"baiduspider\",\n\t\"duckduckbot\",\n\t\"slurp\",\n\t\"msnbot\",\n\t\"facebot\",\n\t\"twitterbot\",\n\t\"linkedinbot\",\n\t\"whatsapp\",\n\t\"telegrambot\",\n\t\"pingdom\",\n\t\"uptimerobot\",\n\t\"newrelic\",\n\t\"datadog\",\n\t\"statuspage\",\n\t\"site24x7\",\n\t\"applebot\",\n];\n\n/**\n * Broad regex for bot-like UA strings (used only in Layer 3 heuristic).\n * No word boundaries — keywords commonly appear in compound names.\n */\nexport const BOT_LIKE_REGEX: RegExp = /bot|agent|fetch|crawl|spider|search/i;\n","import {\n\tAI_AGENT_UA_PATTERNS,\n\tBOT_LIKE_REGEX,\n\tSIGNATURE_AGENT_DOMAINS,\n\tTRADITIONAL_BOT_PATTERNS,\n} from \"./patterns\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\n/**\n * Detects AI agents from HTTP request headers.\n *\n * Three detection layers (checked in order):\n * 1. Known UA patterns (definitive)\n * 2. Signature-Agent header (definitive, RFC 9421)\n * 3. Missing sec-fetch-mode heuristic (catches unknown bots)\n *\n * Optimizes for recall over precision: serving markdown to a non-AI bot\n * is low-harm; missing an AI agent means a worse experience.\n */\nexport function isAIAgent(request: MinimalRequest): DetectionResult {\n\tconst userAgent = request.headers.get(\"user-agent\");\n\tconst lowerUA = userAgent?.toLowerCase() ?? \"\";\n\n\t// Layer 1: Known UA pattern match\n\tif (lowerUA && AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {\n\t\treturn { detected: true, method: \"ua-match\" };\n\t}\n\n\t// Layer 2: Signature-Agent header (RFC 9421, used by ChatGPT agent)\n\tconst signatureAgent = request.headers.get(\"signature-agent\");\n\tif (signatureAgent) {\n\t\tconst lowerSig = signatureAgent.toLowerCase();\n\t\tif (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {\n\t\t\treturn { detected: true, method: \"signature-agent\" };\n\t\t}\n\t}\n\n\t// Layer 3: Missing browser fingerprint heuristic\n\t// Real browsers (Chrome 76+, Firefox 90+, Safari 16.4+) send sec-fetch-mode\n\t// on navigation requests. Its absence signals a programmatic client.\n\tconst secFetchMode = request.headers.get(\"sec-fetch-mode\");\n\tif (!secFetchMode && lowerUA && BOT_LIKE_REGEX.test(lowerUA)) {\n\t\tconst isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) => lowerUA.includes(pattern));\n\t\tif (!isTraditionalBot) {\n\t\t\treturn { detected: true, method: \"heuristic\" };\n\t\t}\n\t}\n\n\treturn { detected: false, method: null };\n}\n","import { isAIAgent } from \"./detection\";\nimport type { DetectionResult, MinimalRequest } from \"./types\";\n\nconst DEFAULT_MARKDOWN_TYPES = [\"text/markdown\", \"text/x-markdown\"];\n\nexport interface AcceptMarkdownOptions {\n\tmediaTypes?: string[];\n}\n\n/**\n * Check if the request prefers markdown via the Accept header.\n */\nexport function acceptsMarkdown(request: MinimalRequest, options?: AcceptMarkdownOptions): boolean {\n\tconst accept = request.headers.get(\"accept\");\n\tif (!accept) return false;\n\n\tconst types = options?.mediaTypes ?? DEFAULT_MARKDOWN_TYPES;\n\tconst lowerAccept = accept.toLowerCase();\n\treturn types.some((type) => lowerAccept.includes(type));\n}\n\nexport interface ShouldServeMarkdownResult {\n\tserve: boolean;\n\treason: \"agent\" | \"accept-header\" | null;\n\tdetection: DetectionResult;\n}\n\n/**\n * Combines agent detection and content negotiation into one call.\n * Returns whether to serve markdown and why.\n */\nexport function shouldServeMarkdown(\n\trequest: MinimalRequest,\n\toptions?: AcceptMarkdownOptions,\n): ShouldServeMarkdownResult {\n\tconst detection = isAIAgent(request);\n\tif (detection.detected) {\n\t\treturn { serve: true, reason: \"agent\", detection };\n\t}\n\n\tif (acceptsMarkdown(request, options)) {\n\t\treturn { serve: true, reason: \"accept-header\", detection };\n\t}\n\n\treturn { serve: false, reason: null, detection };\n}\n"],"mappings":";AACA,SAAS,oBAAoB;;;ACItB,IAAM,uBAA0C;AAAA;AAAA,EAEtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAKO,IAAM,0BAA6C,CAAC,aAAa;AAMjE,IAAM,2BAA8C;AAAA,EAC1D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AAMO,IAAM,iBAAyB;;;ACrE/B,SAAS,UAAU,SAA0C;AACnE,QAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY;AAClD,QAAM,UAAU,WAAW,YAAY,KAAK;AAG5C,MAAI,WAAW,qBAAqB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC,GAAG;AACjF,WAAO,EAAE,UAAU,MAAM,QAAQ,WAAW;AAAA,EAC7C;AAGA,QAAM,iBAAiB,QAAQ,QAAQ,IAAI,iBAAiB;AAC5D,MAAI,gBAAgB;AACnB,UAAM,WAAW,eAAe,YAAY;AAC5C,QAAI,wBAAwB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC,GAAG;AACxE,aAAO,EAAE,UAAU,MAAM,QAAQ,kBAAkB;AAAA,IACpD;AAAA,EACD;AAKA,QAAM,eAAe,QAAQ,QAAQ,IAAI,gBAAgB;AACzD,MAAI,CAAC,gBAAgB,WAAW,eAAe,KAAK,OAAO,GAAG;AAC7D,UAAM,mBAAmB,yBAAyB,KAAK,CAAC,YAAY,QAAQ,SAAS,OAAO,CAAC;AAC7F,QAAI,CAAC,kBAAkB;AACtB,aAAO,EAAE,UAAU,MAAM,QAAQ,YAAY;AAAA,IAC9C;AAAA,EACD;AAEA,SAAO,EAAE,UAAU,OAAO,QAAQ,KAAK;AACxC;;;AC9CA,IAAM,yBAAyB,CAAC,iBAAiB,iBAAiB;AAS3D,SAAS,gBAAgB,SAAyB,SAA0C;AAClG,QAAM,SAAS,QAAQ,QAAQ,IAAI,QAAQ;AAC3C,MAAI,CAAC,OAAQ,QAAO;AAEpB,QAAM,QAAQ,SAAS,cAAc;AACrC,QAAM,cAAc,OAAO,YAAY;AACvC,SAAO,MAAM,KAAK,CAAC,SAAS,YAAY,SAAS,IAAI,CAAC;AACvD;;;AHkBO,SAAS,qBACf,SACA,SACoB;AACpB,SAAO,OAAO,KAAkB,UAA0B;AACzD,UAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAM,SAAS,QAAQ,cAAc;AAErC,QAAI,SAAS,WAAW,MAAM,GAAG;AAChC,YAAM,SAAS,UAAU,GAAG;AAC5B,YAAM,mBAAmB,gBAAgB,GAAG;AAE5C,UAAI,OAAO,YAAY,kBAAkB;AACxC,YAAI,QAAQ,aAAa;AACxB,gBAAM,SAAS,OAAO,WAAW,OAAO,SAAS;AACjD,gBAAM,YAAY,QAAQ,YAAY;AAAA,YACrC,MAAM;AAAA,YACN;AAAA,YACA,WAAW,IAAI,QAAQ,IAAI,YAAY;AAAA,UACxC,CAAC;AACD,cAAI,qBAAqB,SAAS;AACjC,kBAAM,UAAU,SAAS;AAAA,UAC1B;AAAA,QACD;AAEA,cAAM,aAAa,IAAI,QAAQ,MAAM;AACrC,mBAAW,WAAW,QAAQ,QAAQ,QAAQ;AAC9C,eAAO,aAAa,QAAQ,UAAU;AAAA,MACvC;AAAA,IACD;AAEA,QAAI,QAAS,QAAO,QAAQ,KAAK,KAAK;AACtC,WAAO,aAAa,KAAK;AAAA,EAC1B;AACD;AAGO,IAAM,0BAA0B;AAAA,EACtC;AACD;","names":[]}

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "@vercel/agent-readability",
+  "version": "0.1.0",
+  "description": "Detect AI agents. Serve them markdown. Audit your site against the Agent Readability Spec.",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./next": {
+      "import": {
+        "types": "./dist/next/index.d.ts",
+        "default": "./dist/next/index.js"
+      },
+      "require": {
+        "types": "./dist/next/index.d.cts",
+        "default": "./dist/next/index.cjs"
+      }
+    }
+  },
+  "bin": {
+    "agent-readability": "./dist/cli/index.cjs"
+  },
+  "files": [
+    "dist",
+    "skill"
+  ],
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "peerDependencies": {
+    "next": ">=14"
+  },
+  "peerDependenciesMeta": {
+    "next": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.9",
+    "@changesets/cli": "^2",
+    "@types/node": "^22",
+    "@vitest/coverage-v8": "^3",
+    "cheerio": "^1",
+    "citty": "^0.1",
+    "next": "^15",
+    "p-limit": "^6",
+    "picocolors": "^1",
+    "tsup": "^8",
+    "typescript": "^5.8",
+    "vitest": "^3"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vercel-labs/agent-readability.git"
+  },
+  "keywords": [
+    "ai",
+    "agent",
+    "readability",
+    "markdown",
+    "detection",
+    "next.js",
+    "middleware"
+  ],
+  "scripts": {
+    "build": "rm -rf dist && tsup",
+    "test": "vitest",
+    "test:ci": "vitest run --coverage",
+    "lint": "biome check .",
+    "lint:fix": "biome check --write .",
+    "typecheck": "tsc --noEmit",
+    "changeset": "changeset",
+    "release": "changeset publish"
+  }
+}