@agentstats/sdk 0.5.0 → 0.5.1

package/dist/server.cjs

@@ -2,167 +2,8 @@ Object.defineProperties(exports, {
 	__esModule: { value: true },
 	[Symbol.toStringTag]: { value: "Module" }
 });
-//#region src/bots.ts
-/** Categories that count as "AI traffic". Search engines and other bots are shown separately. */
-const AI_CATEGORIES = [
-	"training",
-	"search",
-	"assistant",
-	"agent"
-];
-const isAiCategory = (c) => AI_CATEGORIES.includes(c);
-/** Products group several bots from one vendor ("ChatGPT" = GPTBot + OAI-SearchBot + ChatGPT-User + ChatGPT agent). */
-const BOT_PRODUCTS = {
-	chatgpt: {
-		name: "ChatGPT",
-		operator: "OpenAI"
-	},
-	claude: {
-		name: "Claude",
-		operator: "Anthropic"
-	},
-	perplexity: {
-		name: "Perplexity",
-		operator: "Perplexity"
-	},
-	gemini: {
-		name: "Gemini",
-		operator: "Google"
-	},
-	copilot: {
-		name: "Copilot & Bing",
-		operator: "Microsoft"
-	},
-	meta: {
-		name: "Meta AI",
-		operator: "Meta"
-	},
-	apple: {
-		name: "Apple",
-		operator: "Apple"
-	},
-	amazon: {
-		name: "Amazon",
-		operator: "Amazon"
-	},
-	mistral: {
-		name: "Mistral",
-		operator: "Mistral AI"
-	},
-	deepseek: {
-		name: "DeepSeek",
-		operator: "DeepSeek"
-	},
-	bytedance: {
-		name: "ByteDance",
-		operator: "ByteDance"
-	},
-	xai: {
-		name: "Grok",
-		operator: "xAI"
-	},
-	duckduckgo: {
-		name: "DuckDuckGo",
-		operator: "DuckDuckGo"
-	},
-	commoncrawl: {
-		name: "Common Crawl",
-		operator: "Common Crawl Foundation"
-	},
-	cohere: {
-		name: "Cohere",
-		operator: "Cohere"
-	},
-	you: {
-		name: "You.com",
-		operator: "You.com"
-	},
-	qwen: {
-		name: "Qwen",
-		operator: "Alibaba"
-	},
-	kimi: {
-		name: "Kimi",
-		operator: "Moonshot AI"
-	},
-	baidu: {
-		name: "Baidu",
-		operator: "Baidu"
-	},
-	yandex: {
-		name: "Yandex",
-		operator: "Yandex"
-	},
-	huawei: {
-		name: "Huawei",
-		operator: "Huawei"
-	},
-	ai2: {
-		name: "Ai2",
-		operator: "Allen Institute for AI"
-	},
-	brave: {
-		name: "Brave",
-		operator: "Brave"
-	},
-	kagi: {
-		name: "Kagi",
-		operator: "Kagi"
-	},
-	phind: {
-		name: "Phind",
-		operator: "Phind"
-	},
-	poe: {
-		name: "Poe",
-		operator: "Quora"
-	},
-	manus: {
-		name: "Manus",
-		operator: "Butterfly Effect"
-	},
-	coding: {
-		name: "Coding agents",
-		operator: "Various"
-	},
-	aidata: {
-		name: "AI data providers",
-		operator: "Various"
-	},
-	google: {
-		name: "Google Search",
-		operator: "Google"
-	},
-	automation: {
-		name: "Automated browsers",
-		operator: "Various"
-	},
-	seo: {
-		name: "SEO tools",
-		operator: "Various"
-	},
-	social: {
-		name: "Link previews",
-		operator: "Various"
-	},
-	monitor: {
-		name: "Uptime monitors",
-		operator: "Various"
-	},
-	tools: {
-		name: "HTTP clients & scripts",
-		operator: "Various"
-	},
-	other: {
-		name: "Other bots",
-		operator: "Unknown"
-	}
-};
-/**
-* Order matters: the first matching pattern wins, so specific tokens come
-* before the vendor's generic ones and the catch-all patterns come last.
-*/
-const BOT_PATTERNS = [
+// src/bots.curated.ts
+const CURATED = [
 	{
 		id: "chatgpt-agent",
 		name: "ChatGPT agent",
@@ -231,7 +72,7 @@ const BOT_PATTERNS = [
 		name: "Perplexity-User",
 		product: "perplexity",
 		category: "assistant",
-		pattern: /Perplexity-User/i
+		pattern: /Perplexity-?User/i
 	},
 	{
 		id: "perplexitybot",
@@ -639,6 +480,13 @@ const BOT_PATTERNS = [
 		category: "assistant",
 		pattern: /FirecrawlAgent|Firecrawl/i
 	},
+	{
+		id: "exa-user",
+		name: "Exa (live fetch)",
+		product: "aidata",
+		category: "assistant",
+		pattern: /ExaUser/i
+	},
 	{
 		id: "exabot",
 		name: "Exa",
@@ -737,12 +585,19 @@ const BOT_PATTERNS = [
 		category: "other",
 		pattern: /HeadlessChrome|PhantomJS|Puppeteer|Playwright|Selenium|Lightpanda|webdriver/i
 	},
+	{
+		id: "ahrefs",
+		name: "Ahrefs",
+		product: "seo",
+		category: "other",
+		pattern: /AhrefsBot|AhrefsSiteAudit/i
+	},
 	{
 		id: "seo",
 		name: "SEO crawler",
 		product: "seo",
 		category: "other",
-		pattern: /AhrefsBot|AhrefsSiteAudit|SemrushBot|SiteAuditBot|MJ12bot|DotBot|rogerbot|Screaming Frog|DataForSeoBot|BLEXBot|serpstatbot|SEOkicks|barkrowler/i
+		pattern: /SemrushBot|SiteAuditBot|MJ12bot|DotBot|rogerbot|Screaming Frog|DataForSeoBot|BLEXBot|serpstatbot|SEOkicks|barkrowler/i
 	},
 	{
 		id: "link-preview",
@@ -771,36 +626,234 @@ const BOT_PATTERNS = [
 		product: "tools",
 		category: "other",
 		pattern: /^curl\/|\bWget\b|python-requests|python-urllib|Python\/\d|aiohttp|python-httpx|Go-http-client|okhttp|axios\/|node-fetch|undici|^node$|Java\/\d|Apache-HttpClient|libwww-perl|^Ruby|Faraday|Scrapy|^Deno\/|^Bun\/|GuzzleHttp|^PostmanRuntime|HTTPie/i
+	}
+];
+/** Matched last, after the generated lists: anything that still calls itself a bot, and no User-Agent at all. */
+const CATCH_ALL = [{
+	id: "other-bot",
+	name: "Other bot",
+	product: "other",
+	category: "other",
+	pattern: /bot\b|bot\/|robot|crawl|spider|scrape|scraping|fetcher|preview|archiver|indexer/i
+}, {
+	id: "no-ua",
+	name: "No user agent",
+	product: "other",
+	category: "other",
+	pattern: null
+}];
+// src/bots.ts
+/**
+* Known bots, AI crawlers and AI agents, recognised by their User-Agent: the ones we've
+* checked against each vendor's documentation (bots.curated.ts). `detectBot()` uses them in
+* your app, and the middleware uses them to decide what to send.
+*
+* The AgentStats server makes the final call with a much bigger catalogue (these plus public
+* bot lists), so the dashboard knows bots this SDK version doesn't, without an upgrade.
+*/
+/** Categories that count as "AI traffic". Search engines and other bots are shown separately. */
+const AI_CATEGORIES = [
+	"training",
+	"search",
+	"assistant",
+	"agent"
+];
+const isAiCategory = (c) => AI_CATEGORIES.includes(c);
+/** Products group several bots from one vendor ("ChatGPT" = GPTBot + OAI-SearchBot + ChatGPT-User + ChatGPT agent). */
+const BOT_PRODUCTS = {
+	chatgpt: {
+		name: "ChatGPT",
+		operator: "OpenAI"
 	},
-	{
-		id: "other-bot",
-		name: "Other bot",
-		product: "other",
-		category: "other",
-		pattern: /bot\b|bot\/|robot|crawl|spider|scrape|scraping|fetcher|preview|archiver|indexer/i
+	claude: {
+		name: "Claude",
+		operator: "Anthropic"
 	},
-	{
-		id: "no-ua",
-		name: "No user agent",
-		product: "other",
-		category: "other",
-		pattern: null
+	perplexity: {
+		name: "Perplexity",
+		operator: "Perplexity"
+	},
+	gemini: {
+		name: "Gemini",
+		operator: "Google"
+	},
+	copilot: {
+		name: "Copilot & Bing",
+		operator: "Microsoft"
+	},
+	meta: {
+		name: "Meta AI",
+		operator: "Meta"
+	},
+	apple: {
+		name: "Apple",
+		operator: "Apple"
+	},
+	amazon: {
+		name: "Amazon",
+		operator: "Amazon"
+	},
+	mistral: {
+		name: "Mistral",
+		operator: "Mistral AI"
+	},
+	deepseek: {
+		name: "DeepSeek",
+		operator: "DeepSeek"
+	},
+	bytedance: {
+		name: "ByteDance",
+		operator: "ByteDance"
+	},
+	xai: {
+		name: "Grok",
+		operator: "xAI"
+	},
+	duckduckgo: {
+		name: "DuckDuckGo",
+		operator: "DuckDuckGo"
+	},
+	commoncrawl: {
+		name: "Common Crawl",
+		operator: "Common Crawl Foundation"
+	},
+	cohere: {
+		name: "Cohere",
+		operator: "Cohere"
+	},
+	you: {
+		name: "You.com",
+		operator: "You.com"
+	},
+	qwen: {
+		name: "Qwen",
+		operator: "Alibaba"
+	},
+	kimi: {
+		name: "Kimi",
+		operator: "Moonshot AI"
+	},
+	baidu: {
+		name: "Baidu",
+		operator: "Baidu"
+	},
+	yandex: {
+		name: "Yandex",
+		operator: "Yandex"
+	},
+	huawei: {
+		name: "Huawei",
+		operator: "Huawei"
+	},
+	ai2: {
+		name: "Ai2",
+		operator: "Allen Institute for AI"
+	},
+	brave: {
+		name: "Brave",
+		operator: "Brave"
+	},
+	kagi: {
+		name: "Kagi",
+		operator: "Kagi"
+	},
+	phind: {
+		name: "Phind",
+		operator: "Phind"
+	},
+	poe: {
+		name: "Poe",
+		operator: "Quora"
+	},
+	manus: {
+		name: "Manus",
+		operator: "Butterfly Effect"
+	},
+	ai: {
+		name: "Other AI bots",
+		operator: "Various"
+	},
+	coding: {
+		name: "Coding agents",
+		operator: "Various"
+	},
+	aidata: {
+		name: "AI data providers",
+		operator: "Various"
+	},
+	google: {
+		name: "Google Search",
+		operator: "Google"
+	},
+	searchengines: {
+		name: "Other search engines",
+		operator: "Various"
+	},
+	automation: {
+		name: "Automated browsers",
+		operator: "Various"
+	},
+	seo: {
+		name: "SEO tools",
+		operator: "Various"
+	},
+	social: {
+		name: "Link previews",
+		operator: "Various"
+	},
+	monitor: {
+		name: "Monitoring",
+		operator: "Various"
+	},
+	feeds: {
+		name: "Feed readers",
+		operator: "Various"
+	},
+	security: {
+		name: "Security scanners",
+		operator: "Various"
+	},
+	archive: {
+		name: "Archivers",
+		operator: "Various"
+	},
+	research: {
+		name: "Research crawlers",
+		operator: "Various"
+	},
+	ads: {
+		name: "Advertising bots",
+		operator: "Various"
+	},
+	tools: {
+		name: "HTTP clients & scripts",
+		operator: "Various"
+	},
+	other: {
+		name: "Other bots",
+		operator: "Unknown"
 	}
-];
+};
+/** Every bot this SDK recognises, in the order they're tried: the first match wins, catch-alls last. */
+const BOT_PATTERNS = [...CURATED, ...CATCH_ALL];
 const BY_ID = new Map(BOT_PATTERNS.map((b) => [b.id, b]));
 /** Real browsers whose UA happens to contain a bot-ish word. */
 const HUMAN_LOOKALIKES = /cubot|\bbotswana\b/i;
-/** Identify a bot from its User-Agent. Returns null for (what looks like) a normal browser. */
-function classifyUserAgent(ua) {
-	const s = (ua ?? "").trim();
-	if (!s) return BY_ID.get("no-ua");
-	for (const b of BOT_PATTERNS) {
+/** The first bot in `list` whose pattern matches `ua` (trimmed, non-empty). Shared with the server's bigger catalogue. */
+function firstMatch(list, s) {
+	for (const b of list) {
 		if (!b.pattern || !b.pattern.test(s)) continue;
 		if ((b.id === "other-bot" || b.id === "link-preview") && HUMAN_LOOKALIKES.test(s)) continue;
 		return b;
 	}
 	return null;
 }
+/** Identify a bot from its User-Agent. Returns null for (what looks like) a normal browser. */
+function classifyUserAgent(ua) {
+	const s = (ua ?? "").trim();
+	if (!s) return BY_ID.get("no-ua");
+	return firstMatch(BOT_PATTERNS, s);
+}
 /** Hostnames of AI assistants that send people to websites, and the `utm_source` values they add. */
 const AI_REFERRERS = [
 	{
@@ -952,14 +1005,13 @@ function detectFromHeaders(header, url) {
 		name: bot.name,
 		category: bot.category,
 		product: product.name,
-		operator: product.operator,
-		browser: bot.category === "agent" || bot.id === "headless",
+		operator: bot.operator ?? product.operator,
+		browser: bot.category === "agent" || bot.product === "automation",
 		signed,
 		referral: null
 	};
 }
-//#endregion
-//#region src/server.ts
+// src/server.ts
 /**
 * AgentStats server integration — see which AI crawlers and agents visit your
 * site (GPTBot, ClaudeBot, PerplexityBot, ChatGPT agent, Google-Agent, …),
@@ -993,12 +1045,21 @@ function detectFromHeaders(header, url) {
 *
 *   const lens = createAgentStats({ apiKey: process.env.AGENTSTATS_API_KEY }) // API-only server
 */
-const SERVER_SDK_VERSION = "0.3.1";
+const SERVER_SDK_VERSION = "0.3.2";
 /**
-* Words that may mean a bot the table below doesn't know yet. Deliberately broad: the server
-* makes the final call, and its table can be newer than this SDK's.
+* Words that may mean a bot this SDK's table doesn't know. Deliberately broad: the server makes
+* the final call, with a catalogue far bigger and newer than this SDK's.
 */
 const BOT_HINT = /bot|crawl|spider|slurp|scrap|fetch|agent|preview|headless|python|curl|wget|httpx|http-?client|go-http|java\/|okhttp|axios|node|undici|libwww|perl|ruby|phantom|puppeteer|playwright|selenium|lighthouse|monitor|uptime|gpt|openai|claude|anthropic|perplexity|gemini|google-|googleother|meta-external|facebookexternalhit|bytespider|ccbot|diffbot|cohere|mistral|deepseek|manus|firecrawl|jina|tavily|exabot|linkup|novaact|amzn|cursor|devin|opencode|kagi|duckassist|qwen|tongyi|kimi|yiyan|ernie|doubao|petal|pangu|timpi|omgili|webzio|imagesift|img2dataset|lightpanda|crawl4ai|apify|youbot|phind/i;
+/**
+* A person's browser: "Mozilla/5.0 (…) AppleWebKit/… or Gecko/…", with no contact URL, email or
+* "compatible;", which bots dressed as browsers add and browsers never do. Anything else is sent
+* for the server to classify with its much bigger catalogue (crawlers that name themselves without
+* a bot-ish word, feed readers, scripts); what it can't name it counts as a person and keeps nothing.
+*/
+const BROWSER_SHAPED = /^Mozilla\/5\.0 \(.*(?:AppleWebKit\/|Gecko\/)/;
+const NOT_A_BROWSER = /compatible;|https?:\/\/|@/i;
+const looksLikeBrowser = (ua) => BROWSER_SHAPED.test(ua) && !NOT_A_BROWSER.test(ua);
 const ASSET = /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|bmp|woff2?|ttf|otf|eot|mp4|webm|mov|mp3|wav|ogg|zip|gz|wasm|pdf)$/i;
 const SIGNATURE_HEADERS = [
 	"signature",
@@ -1034,6 +1095,7 @@ function normalizeRoutePattern(pattern) {
 function fitRoute(pattern, path) {
 	if (typeof pattern !== "string" || !pattern.startsWith("/")) return null;
 	const p = normalizeRoutePattern(pattern).replace(/\/$/, "") || "/";
+	if (/^\/(?:\*\*?|:[^/]+\*)(?::[^/]*)?$/.test(p)) return null;
 	if (p.includes("*")) return p;
 	const segments = (s) => s.split("/").filter(Boolean).length;
 	return segments(p) === segments(path) ? p : null;
@@ -1133,7 +1195,7 @@ var AgentStatsServer = class {
 		if (opts.serverKey && !opts.serverKey.startsWith("al_srv_")) throw new Error("[agentstats] `serverKey` must be the secret server key (al_srv_…), not the public site key");
 		if (opts.apiKey && !opts.apiKey.startsWith("al_api_")) throw new Error("[agentstats] `apiKey` must be an API's key (al_api_…, Project settings → APIs)");
 		const endpoint = opts?.endpoint ?? defaultEndpoint();
-		if (!endpoint && (opts?.serverKey || opts?.apiKey) && !opts?.disabled) throw new Error("[agentstats] `endpoint` is required, e.g. https://your-host/api/ingest/hits");
+		if (!endpoint && (opts?.serverKey || opts?.apiKey) && !opts?.disabled) throw new Error("[agentstats] `endpoint` is required, e.g. https://agentstats.io/api/ingest/hits");
 		this.options = {
 			countHumans: true,
 			visitCookie: true,
@@ -1183,7 +1245,18 @@ var AgentStatsServer = class {
 	* so the hit is sent after the response. Returns what `detectBot` says about the request.
 	*/
 	track(request, init = {}) {
-		const d = this.fetchDecision(request, init.ip);
+		let ip = init.ip;
+		if (ip === void 0 && init.remoteAddress !== void 0) try {
+			const headers = {};
+			request.headers.forEach((v, k) => {
+				headers[k] = v;
+			});
+			ip = this.nodeIp({
+				headers,
+				socket: { remoteAddress: init.remoteAddress ?? void 0 }
+			});
+		} catch {}
+		const d = this.fetchDecision(request, ip);
 		this.setRoute(d, init.route);
 		this.fetchFinish(d, init.status ?? null, init.response, init.waitUntil);
 		return d.detection;
@@ -1253,9 +1326,10 @@ var AgentStatsServer = class {
 		};
 	}
 	/**
-	* Nuxt / Nitro plugin: `export default defineNitroPlugin(lens.nitro())` in `server/plugins/agentstats.ts`.
+	* Nuxt / Nitro plugin: `export default defineNitroPlugin(lens.nitro())` in `server/plugins/agentstats.ts`
+	* (Nitro v3: `export default definePlugin(lens.nitro())`).
 	* The detection is on `event.context.bot` in server routes, and on `useRequestEvent()?.context.bot` in pages.
-	* Works on every Nitro preset (Node, Vercel, Netlify, Cloudflare…).
+	* Works on every Nitro preset (Node, Vercel, Netlify, Cloudflare…), on Nitro v2 (h3 v1) and v3 (h3 v2).
 	*/
 	nitro() {
 		return (nitroApp) => {
@@ -1264,18 +1338,27 @@ var AgentStatsServer = class {
 				event.context.bot = d.detection;
 				if (d.kind === "ignore") return;
 				event.context[DECISION] = d;
-				if (d.cookie && event.node?.res) appendNodeCookie(event.node.res, d.cookie);
+				if (d.cookie) {
+					if (isH3v2(event)) try {
+						event.res.headers.append("set-cookie", d.cookie);
+					} catch {}
+					else if (event.node?.res) appendNodeCookie(event.node.res, d.cookie);
+				}
 			});
-			const finish = (event, status) => {
+			const finish = (event, status, contentType) => {
 				const d = event?.context[DECISION];
 				if (!d) return;
 				delete event.context[DECISION];
-				this.setRoute(d, event.context.matchedRoute?.path);
-				this.setContentType(d, event.node?.res?.getHeader?.("content-type"));
+				const matched = event.context.matchedRoute;
+				this.setRoute(d, matched?.path ?? matched?.route);
+				this.setContentType(d, contentType);
 				this.commit(d, status, typeof event.waitUntil === "function" ? event.waitUntil.bind(event) : void 0);
 			};
-			nitroApp.hooks.hook("afterResponse", (event) => finish(event, event.node?.res?.statusCode ?? null));
-			nitroApp.hooks.hook("error", (error, context = {}) => finish(context.event, error?.statusCode ?? 500));
+			nitroApp.hooks.hook("afterResponse", (event) => finish(event, event.node?.res?.statusCode ?? null, event.node?.res?.getHeader?.("content-type")));
+			nitroApp.hooks.hook("response", (res, event) => finish(event, res?.status ?? event?.res?.status ?? null, res?.headers?.get?.("content-type")));
+			nitroApp.hooks.hook("error", (error, context = {}) => {
+				if (context.event && !isH3v2(context.event)) finish(context.event, error?.statusCode ?? error?.status ?? 500, context.event.node?.res?.getHeader?.("content-type"));
+			});
 		};
 	}
 	/**
@@ -1413,7 +1496,7 @@ var AgentStatsServer = class {
 		const ua = header("user-agent") ?? null;
 		if (!ip && !ua?.trim()) return ignore;
 		const signed = detection.signed;
-		if (signed || detection.isBot || ua && BOT_HINT.test(ua)) {
+		if (signed || detection.isBot || ua && (BOT_HINT.test(ua) || !looksLikeBrowser(ua))) {
 			const record = {
 				ts: Date.now(),
 				method: m,
@@ -1578,15 +1661,18 @@ var AgentStatsServer = class {
 	}
 	h3Decision(event) {
 		const req = event.node?.req;
-		const get = typeof event.headers?.get === "function" ? (name) => event.headers.get(name) : (name) => headerValue(req?.headers[name]);
+		const web = isH3v2(event) ? event.req : void 0;
+		const get = web ? (name) => web.headers.get(name) : typeof event.headers?.get === "function" ? (name) => event.headers.get(name) : (name) => headerValue(req?.headers[name]);
 		try {
-			if (get("x-nitro-prerender") || (event.path ?? req?.url ?? "").startsWith("/__nuxt_error")) return {
+			const path = event.url?.pathname ?? event.path ?? req?.url ?? "";
+			if (get("x-nitro-prerender") || path.startsWith("/__nuxt_error")) return {
 				kind: "ignore",
 				detection: detectFromHeaders(get, null)
 			};
 			const native = !!req?.socket?.remoteAddress;
-			const url = native && req ? this.nodeUrl(req) : event.web?.request?.url ?? `${firstIp(get("x-forwarded-proto")) || "https"}://${get("host") ?? "localhost"}${event.path ?? req?.url ?? "/"}`;
-			return this.decide(event.method ?? req?.method ?? "GET", url, get, native && req ? this.nodeIp(req) : this.fetchIp(get));
+			const url = native && req ? this.nodeUrl(req) : web?.url ?? event.web?.request?.url ?? `${firstIp(get("x-forwarded-proto")) || "https"}://${get("host") ?? "localhost"}${event.path ?? req?.url ?? "/"}`;
+			const ip = native && req ? this.nodeIp(req) : typeof web?.ip === "string" && web.ip && !this.options.ipHeader && !this.options.trustProxy && !platformIpHeader() ? web.ip : this.fetchIp(get);
+			return this.decide(web?.method ?? event.method ?? req?.method ?? "GET", url, get, ip);
 		} catch (err) {
 			this.log("track failed", err);
 			return {
@@ -1625,6 +1711,10 @@ var AgentStatsServer = class {
 	}
 };
 const DECISION = Symbol.for("agentstats.decision");
+/** An h3 v2 event (Nitro v3), whose request is a web `Request`. h3 v1 events have no `req`. */
+function isH3v2(event) {
+	return typeof event.req?.headers?.get === "function";
+}
 function headerValue(v) {
 	if (v === void 0) return null;
 	return Array.isArray(v) ? v.join(", ") : v;
@@ -1638,7 +1728,6 @@ function appendNodeCookie(res, cookie) {
 function createAgentStats(options) {
 	return new AgentStatsServer(options);
 }
-//#endregion
 exports.AI_CATEGORIES = AI_CATEGORIES;
 exports.AgentStatsServer = AgentStatsServer;
 exports.BOT_PATTERNS = BOT_PATTERNS;