@agentstats/sdk 0.0.0-stage → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +70 -2
- package/dist/browser.iife.js +1 -0
- package/dist/index.cjs +800 -0
- package/dist/index.d.cts +269 -0
- package/dist/index.d.ts +269 -0
- package/dist/index.js +786 -0
- package/dist/lite.iife.js +1 -0
- package/dist/server.cjs +1655 -0
- package/dist/server.d.cts +516 -0
- package/dist/server.d.ts +516 -0
- package/dist/server.js +1638 -0
- package/package.json +53 -4
package/dist/server.js
ADDED
|
@@ -0,0 +1,1638 @@
|
|
|
1
|
+
//#region src/bots.ts
|
|
2
|
+
/** Categories that count as "AI traffic". Search engines and other bots are shown separately. */
|
|
3
|
+
const AI_CATEGORIES = [
|
|
4
|
+
"training",
|
|
5
|
+
"search",
|
|
6
|
+
"assistant",
|
|
7
|
+
"agent"
|
|
8
|
+
];
|
|
9
|
+
const isAiCategory = (c) => AI_CATEGORIES.includes(c);
|
|
10
|
+
/** Products group several bots from one vendor ("ChatGPT" = GPTBot + OAI-SearchBot + ChatGPT-User + ChatGPT agent). */
|
|
11
|
+
const BOT_PRODUCTS = {
|
|
12
|
+
chatgpt: {
|
|
13
|
+
name: "ChatGPT",
|
|
14
|
+
operator: "OpenAI"
|
|
15
|
+
},
|
|
16
|
+
claude: {
|
|
17
|
+
name: "Claude",
|
|
18
|
+
operator: "Anthropic"
|
|
19
|
+
},
|
|
20
|
+
perplexity: {
|
|
21
|
+
name: "Perplexity",
|
|
22
|
+
operator: "Perplexity"
|
|
23
|
+
},
|
|
24
|
+
gemini: {
|
|
25
|
+
name: "Gemini",
|
|
26
|
+
operator: "Google"
|
|
27
|
+
},
|
|
28
|
+
copilot: {
|
|
29
|
+
name: "Copilot & Bing",
|
|
30
|
+
operator: "Microsoft"
|
|
31
|
+
},
|
|
32
|
+
meta: {
|
|
33
|
+
name: "Meta AI",
|
|
34
|
+
operator: "Meta"
|
|
35
|
+
},
|
|
36
|
+
apple: {
|
|
37
|
+
name: "Apple",
|
|
38
|
+
operator: "Apple"
|
|
39
|
+
},
|
|
40
|
+
amazon: {
|
|
41
|
+
name: "Amazon",
|
|
42
|
+
operator: "Amazon"
|
|
43
|
+
},
|
|
44
|
+
mistral: {
|
|
45
|
+
name: "Mistral",
|
|
46
|
+
operator: "Mistral AI"
|
|
47
|
+
},
|
|
48
|
+
deepseek: {
|
|
49
|
+
name: "DeepSeek",
|
|
50
|
+
operator: "DeepSeek"
|
|
51
|
+
},
|
|
52
|
+
bytedance: {
|
|
53
|
+
name: "ByteDance",
|
|
54
|
+
operator: "ByteDance"
|
|
55
|
+
},
|
|
56
|
+
xai: {
|
|
57
|
+
name: "Grok",
|
|
58
|
+
operator: "xAI"
|
|
59
|
+
},
|
|
60
|
+
duckduckgo: {
|
|
61
|
+
name: "DuckDuckGo",
|
|
62
|
+
operator: "DuckDuckGo"
|
|
63
|
+
},
|
|
64
|
+
commoncrawl: {
|
|
65
|
+
name: "Common Crawl",
|
|
66
|
+
operator: "Common Crawl Foundation"
|
|
67
|
+
},
|
|
68
|
+
cohere: {
|
|
69
|
+
name: "Cohere",
|
|
70
|
+
operator: "Cohere"
|
|
71
|
+
},
|
|
72
|
+
you: {
|
|
73
|
+
name: "You.com",
|
|
74
|
+
operator: "You.com"
|
|
75
|
+
},
|
|
76
|
+
qwen: {
|
|
77
|
+
name: "Qwen",
|
|
78
|
+
operator: "Alibaba"
|
|
79
|
+
},
|
|
80
|
+
kimi: {
|
|
81
|
+
name: "Kimi",
|
|
82
|
+
operator: "Moonshot AI"
|
|
83
|
+
},
|
|
84
|
+
baidu: {
|
|
85
|
+
name: "Baidu",
|
|
86
|
+
operator: "Baidu"
|
|
87
|
+
},
|
|
88
|
+
yandex: {
|
|
89
|
+
name: "Yandex",
|
|
90
|
+
operator: "Yandex"
|
|
91
|
+
},
|
|
92
|
+
huawei: {
|
|
93
|
+
name: "Huawei",
|
|
94
|
+
operator: "Huawei"
|
|
95
|
+
},
|
|
96
|
+
ai2: {
|
|
97
|
+
name: "Ai2",
|
|
98
|
+
operator: "Allen Institute for AI"
|
|
99
|
+
},
|
|
100
|
+
brave: {
|
|
101
|
+
name: "Brave",
|
|
102
|
+
operator: "Brave"
|
|
103
|
+
},
|
|
104
|
+
kagi: {
|
|
105
|
+
name: "Kagi",
|
|
106
|
+
operator: "Kagi"
|
|
107
|
+
},
|
|
108
|
+
phind: {
|
|
109
|
+
name: "Phind",
|
|
110
|
+
operator: "Phind"
|
|
111
|
+
},
|
|
112
|
+
poe: {
|
|
113
|
+
name: "Poe",
|
|
114
|
+
operator: "Quora"
|
|
115
|
+
},
|
|
116
|
+
manus: {
|
|
117
|
+
name: "Manus",
|
|
118
|
+
operator: "Butterfly Effect"
|
|
119
|
+
},
|
|
120
|
+
coding: {
|
|
121
|
+
name: "Coding agents",
|
|
122
|
+
operator: "Various"
|
|
123
|
+
},
|
|
124
|
+
aidata: {
|
|
125
|
+
name: "AI data providers",
|
|
126
|
+
operator: "Various"
|
|
127
|
+
},
|
|
128
|
+
google: {
|
|
129
|
+
name: "Google Search",
|
|
130
|
+
operator: "Google"
|
|
131
|
+
},
|
|
132
|
+
automation: {
|
|
133
|
+
name: "Automated browsers",
|
|
134
|
+
operator: "Various"
|
|
135
|
+
},
|
|
136
|
+
seo: {
|
|
137
|
+
name: "SEO tools",
|
|
138
|
+
operator: "Various"
|
|
139
|
+
},
|
|
140
|
+
social: {
|
|
141
|
+
name: "Link previews",
|
|
142
|
+
operator: "Various"
|
|
143
|
+
},
|
|
144
|
+
monitor: {
|
|
145
|
+
name: "Uptime monitors",
|
|
146
|
+
operator: "Various"
|
|
147
|
+
},
|
|
148
|
+
tools: {
|
|
149
|
+
name: "HTTP clients & scripts",
|
|
150
|
+
operator: "Various"
|
|
151
|
+
},
|
|
152
|
+
other: {
|
|
153
|
+
name: "Other bots",
|
|
154
|
+
operator: "Unknown"
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
/**
|
|
158
|
+
* Order matters: the first matching pattern wins, so specific tokens come
|
|
159
|
+
* before the vendor's generic ones and the catch-all patterns come last.
|
|
160
|
+
*/
|
|
161
|
+
const BOT_PATTERNS = [
|
|
162
|
+
{
|
|
163
|
+
id: "chatgpt-agent",
|
|
164
|
+
name: "ChatGPT agent",
|
|
165
|
+
product: "chatgpt",
|
|
166
|
+
category: "agent",
|
|
167
|
+
pattern: /ChatGPT[- ]Agent/i
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
id: "chatgpt-user",
|
|
171
|
+
name: "ChatGPT-User",
|
|
172
|
+
product: "chatgpt",
|
|
173
|
+
category: "assistant",
|
|
174
|
+
pattern: /ChatGPT-User/i
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
id: "oai-searchbot",
|
|
178
|
+
name: "OAI-SearchBot",
|
|
179
|
+
product: "chatgpt",
|
|
180
|
+
category: "search",
|
|
181
|
+
pattern: /OAI-SearchBot/i
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
id: "gptbot",
|
|
185
|
+
name: "GPTBot",
|
|
186
|
+
product: "chatgpt",
|
|
187
|
+
category: "training",
|
|
188
|
+
pattern: /GPTBot/i
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
id: "oai-adsbot",
|
|
192
|
+
name: "OAI-AdsBot",
|
|
193
|
+
product: "chatgpt",
|
|
194
|
+
category: "other",
|
|
195
|
+
pattern: /OAI-AdsBot/i
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
id: "claude-code",
|
|
199
|
+
name: "Claude Code",
|
|
200
|
+
product: "claude",
|
|
201
|
+
category: "assistant",
|
|
202
|
+
pattern: /claude-code/i
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
id: "claude-user",
|
|
206
|
+
name: "Claude-User",
|
|
207
|
+
product: "claude",
|
|
208
|
+
category: "assistant",
|
|
209
|
+
pattern: /Claude-User/i
|
|
210
|
+
},
|
|
211
|
+
{
|
|
212
|
+
id: "claude-searchbot",
|
|
213
|
+
name: "Claude-SearchBot",
|
|
214
|
+
product: "claude",
|
|
215
|
+
category: "search",
|
|
216
|
+
pattern: /Claude-SearchBot/i
|
|
217
|
+
},
|
|
218
|
+
{
|
|
219
|
+
id: "claudebot",
|
|
220
|
+
name: "ClaudeBot",
|
|
221
|
+
product: "claude",
|
|
222
|
+
category: "training",
|
|
223
|
+
pattern: /ClaudeBot|Claude-Web|anthropic-ai/i
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
id: "perplexity-user",
|
|
227
|
+
name: "Perplexity-User",
|
|
228
|
+
product: "perplexity",
|
|
229
|
+
category: "assistant",
|
|
230
|
+
pattern: /Perplexity-User/i
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
id: "perplexitybot",
|
|
234
|
+
name: "PerplexityBot",
|
|
235
|
+
product: "perplexity",
|
|
236
|
+
category: "search",
|
|
237
|
+
pattern: /PerplexityBot/i
|
|
238
|
+
},
|
|
239
|
+
{
|
|
240
|
+
id: "google-agent",
|
|
241
|
+
name: "Google-Agent",
|
|
242
|
+
product: "gemini",
|
|
243
|
+
category: "agent",
|
|
244
|
+
pattern: /Google-Agent\b/i
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
id: "googleagent-mariner",
|
|
248
|
+
name: "Project Mariner",
|
|
249
|
+
product: "gemini",
|
|
250
|
+
category: "agent",
|
|
251
|
+
pattern: /GoogleAgent-Mariner/i
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
id: "googleagent-urlcontext",
|
|
255
|
+
name: "Gemini URL context",
|
|
256
|
+
product: "gemini",
|
|
257
|
+
category: "assistant",
|
|
258
|
+
pattern: /GoogleAgent-URLContext/i
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
id: "gemini-deep-research",
|
|
262
|
+
name: "Gemini Deep Research",
|
|
263
|
+
product: "gemini",
|
|
264
|
+
category: "assistant",
|
|
265
|
+
pattern: /Gemini-Deep-Research/i
|
|
266
|
+
},
|
|
267
|
+
{
|
|
268
|
+
id: "google-gemininotebook",
|
|
269
|
+
name: "Gemini Notebook",
|
|
270
|
+
product: "gemini",
|
|
271
|
+
category: "assistant",
|
|
272
|
+
pattern: /Google-GeminiNotebook|Google-NotebookLM|NotebookLM/i
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
id: "gemini-cli",
|
|
276
|
+
name: "Gemini CLI",
|
|
277
|
+
product: "gemini",
|
|
278
|
+
category: "assistant",
|
|
279
|
+
pattern: /Gemini-?CLI/i
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
id: "google-cloudvertexbot",
|
|
283
|
+
name: "Google-CloudVertexBot",
|
|
284
|
+
product: "gemini",
|
|
285
|
+
category: "search",
|
|
286
|
+
pattern: /Google-CloudVertexBot/i
|
|
287
|
+
},
|
|
288
|
+
{
|
|
289
|
+
id: "google-read-aloud",
|
|
290
|
+
name: "Google Read Aloud",
|
|
291
|
+
product: "google",
|
|
292
|
+
category: "other",
|
|
293
|
+
pattern: /Google-Read-Aloud/i
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
id: "google-ads",
|
|
297
|
+
name: "Google Ads crawlers",
|
|
298
|
+
product: "google",
|
|
299
|
+
category: "other",
|
|
300
|
+
pattern: /AdsBot-Google|Mediapartners-Google|APIs-Google/i
|
|
301
|
+
},
|
|
302
|
+
{
|
|
303
|
+
id: "googleother",
|
|
304
|
+
name: "GoogleOther",
|
|
305
|
+
product: "google",
|
|
306
|
+
category: "other",
|
|
307
|
+
pattern: /GoogleOther|Google-InspectionTool|Storebot-Google/i
|
|
308
|
+
},
|
|
309
|
+
{
|
|
310
|
+
id: "googlebot",
|
|
311
|
+
name: "Googlebot",
|
|
312
|
+
product: "google",
|
|
313
|
+
category: "search_engine",
|
|
314
|
+
pattern: /Googlebot/i
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
id: "azureai-searchbot",
|
|
318
|
+
name: "AzureAI-SearchBot",
|
|
319
|
+
product: "copilot",
|
|
320
|
+
category: "search",
|
|
321
|
+
pattern: /AzureAI-SearchBot/i
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
id: "bingbot",
|
|
325
|
+
name: "Bingbot",
|
|
326
|
+
product: "copilot",
|
|
327
|
+
category: "search_engine",
|
|
328
|
+
pattern: /bingbot|BingPreview|msnbot/i
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
id: "meta-externalfetcher",
|
|
332
|
+
name: "Meta-ExternalFetcher",
|
|
333
|
+
product: "meta",
|
|
334
|
+
category: "assistant",
|
|
335
|
+
pattern: /meta-externalfetcher/i
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
id: "meta-webindexer",
|
|
339
|
+
name: "Meta-WebIndexer",
|
|
340
|
+
product: "meta",
|
|
341
|
+
category: "search",
|
|
342
|
+
pattern: /meta-webindexer/i
|
|
343
|
+
},
|
|
344
|
+
{
|
|
345
|
+
id: "meta-externalagent",
|
|
346
|
+
name: "Meta-ExternalAgent",
|
|
347
|
+
product: "meta",
|
|
348
|
+
category: "training",
|
|
349
|
+
pattern: /meta-externalagent|FacebookBot/i
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
id: "facebookexternalhit",
|
|
353
|
+
name: "Facebook link preview",
|
|
354
|
+
product: "meta",
|
|
355
|
+
category: "other",
|
|
356
|
+
pattern: /facebookexternalhit|facebookcatalog/i
|
|
357
|
+
},
|
|
358
|
+
{
|
|
359
|
+
id: "applebot",
|
|
360
|
+
name: "Applebot",
|
|
361
|
+
product: "apple",
|
|
362
|
+
category: "search_engine",
|
|
363
|
+
pattern: /Applebot/i
|
|
364
|
+
},
|
|
365
|
+
{
|
|
366
|
+
id: "amazonbuyforme",
|
|
367
|
+
name: "Amazon Buy for Me",
|
|
368
|
+
product: "amazon",
|
|
369
|
+
category: "agent",
|
|
370
|
+
pattern: /AmazonBuyForMe/i
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
id: "novaact",
|
|
374
|
+
name: "Nova Act",
|
|
375
|
+
product: "amazon",
|
|
376
|
+
category: "agent",
|
|
377
|
+
pattern: /NovaAct/i
|
|
378
|
+
},
|
|
379
|
+
{
|
|
380
|
+
id: "amzn-user",
|
|
381
|
+
name: "Amzn-User",
|
|
382
|
+
product: "amazon",
|
|
383
|
+
category: "assistant",
|
|
384
|
+
pattern: /Amzn-User/i
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
id: "amzn-searchbot",
|
|
388
|
+
name: "Amzn-SearchBot",
|
|
389
|
+
product: "amazon",
|
|
390
|
+
category: "search",
|
|
391
|
+
pattern: /Amzn-SearchBot|amazon-kendra/i
|
|
392
|
+
},
|
|
393
|
+
{
|
|
394
|
+
id: "bedrockbot",
|
|
395
|
+
name: "BedrockBot",
|
|
396
|
+
product: "amazon",
|
|
397
|
+
category: "training",
|
|
398
|
+
pattern: /bedrockbot/i
|
|
399
|
+
},
|
|
400
|
+
{
|
|
401
|
+
id: "amazonbot",
|
|
402
|
+
name: "Amazonbot",
|
|
403
|
+
product: "amazon",
|
|
404
|
+
category: "search",
|
|
405
|
+
pattern: /Amazonbot/i
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
id: "mistralai-user",
|
|
409
|
+
name: "MistralAI-User",
|
|
410
|
+
product: "mistral",
|
|
411
|
+
category: "assistant",
|
|
412
|
+
pattern: /MistralAI-User/i
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
id: "mistralai-index",
|
|
416
|
+
name: "MistralAI-Index",
|
|
417
|
+
product: "mistral",
|
|
418
|
+
category: "search",
|
|
419
|
+
pattern: /MistralAI-Index/i
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
id: "mistralai-training",
|
|
423
|
+
name: "MistralAI-Training",
|
|
424
|
+
product: "mistral",
|
|
425
|
+
category: "training",
|
|
426
|
+
pattern: /MistralAI-Training/i
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
id: "deepseekbot",
|
|
430
|
+
name: "DeepSeekBot",
|
|
431
|
+
product: "deepseek",
|
|
432
|
+
category: "training",
|
|
433
|
+
pattern: /DeepSeekBot/i
|
|
434
|
+
},
|
|
435
|
+
{
|
|
436
|
+
id: "doubaobot",
|
|
437
|
+
name: "DoubaoBot",
|
|
438
|
+
product: "bytedance",
|
|
439
|
+
category: "search",
|
|
440
|
+
pattern: /DoubaoBot/i
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
id: "bytespider",
|
|
444
|
+
name: "Bytespider",
|
|
445
|
+
product: "bytedance",
|
|
446
|
+
category: "training",
|
|
447
|
+
pattern: /Bytespider|TikTokSpider/i
|
|
448
|
+
},
|
|
449
|
+
{
|
|
450
|
+
id: "duckassistbot",
|
|
451
|
+
name: "DuckAssistBot",
|
|
452
|
+
product: "duckduckgo",
|
|
453
|
+
category: "assistant",
|
|
454
|
+
pattern: /DuckAssistBot/i
|
|
455
|
+
},
|
|
456
|
+
{
|
|
457
|
+
id: "duckduckbot",
|
|
458
|
+
name: "DuckDuckBot",
|
|
459
|
+
product: "duckduckgo",
|
|
460
|
+
category: "search_engine",
|
|
461
|
+
pattern: /DuckDuckBot/i
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
id: "ccbot",
|
|
465
|
+
name: "CCBot",
|
|
466
|
+
product: "commoncrawl",
|
|
467
|
+
category: "training",
|
|
468
|
+
pattern: /CCBot/i
|
|
469
|
+
},
|
|
470
|
+
{
|
|
471
|
+
id: "cohere-training",
|
|
472
|
+
name: "Cohere training crawler",
|
|
473
|
+
product: "cohere",
|
|
474
|
+
category: "training",
|
|
475
|
+
pattern: /cohere-training-data-crawler/i
|
|
476
|
+
},
|
|
477
|
+
{
|
|
478
|
+
id: "cohere-ai",
|
|
479
|
+
name: "cohere-ai",
|
|
480
|
+
product: "cohere",
|
|
481
|
+
category: "assistant",
|
|
482
|
+
pattern: /cohere-ai/i
|
|
483
|
+
},
|
|
484
|
+
{
|
|
485
|
+
id: "youbot",
|
|
486
|
+
name: "YouBot",
|
|
487
|
+
product: "you",
|
|
488
|
+
category: "search",
|
|
489
|
+
pattern: /YouBot/i
|
|
490
|
+
},
|
|
491
|
+
{
|
|
492
|
+
id: "tongyibot",
|
|
493
|
+
name: "TongyiBot",
|
|
494
|
+
product: "qwen",
|
|
495
|
+
category: "assistant",
|
|
496
|
+
pattern: /TongyiBot/i
|
|
497
|
+
},
|
|
498
|
+
{
|
|
499
|
+
id: "qwenbot",
|
|
500
|
+
name: "QwenBot",
|
|
501
|
+
product: "qwen",
|
|
502
|
+
category: "training",
|
|
503
|
+
pattern: /QwenBot/i
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
id: "kimi-user",
|
|
507
|
+
name: "Kimi-User",
|
|
508
|
+
product: "kimi",
|
|
509
|
+
category: "assistant",
|
|
510
|
+
pattern: /Kimi-User/i
|
|
511
|
+
},
|
|
512
|
+
{
|
|
513
|
+
id: "kimi-searchbot",
|
|
514
|
+
name: "Kimi-SearchBot",
|
|
515
|
+
product: "kimi",
|
|
516
|
+
category: "search",
|
|
517
|
+
pattern: /Kimi-SearchBot/i
|
|
518
|
+
},
|
|
519
|
+
{
|
|
520
|
+
id: "kimibot",
|
|
521
|
+
name: "KimiBot",
|
|
522
|
+
product: "kimi",
|
|
523
|
+
category: "training",
|
|
524
|
+
pattern: /KimiBot/i
|
|
525
|
+
},
|
|
526
|
+
{
|
|
527
|
+
id: "yiyanbot",
|
|
528
|
+
name: "YiyanBot",
|
|
529
|
+
product: "baidu",
|
|
530
|
+
category: "assistant",
|
|
531
|
+
pattern: /YiyanBot/i
|
|
532
|
+
},
|
|
533
|
+
{
|
|
534
|
+
id: "erniebot",
|
|
535
|
+
name: "ERNIEBot",
|
|
536
|
+
product: "baidu",
|
|
537
|
+
category: "training",
|
|
538
|
+
pattern: /ERNIEBot/i
|
|
539
|
+
},
|
|
540
|
+
{
|
|
541
|
+
id: "baiduspider",
|
|
542
|
+
name: "Baiduspider",
|
|
543
|
+
product: "baidu",
|
|
544
|
+
category: "search_engine",
|
|
545
|
+
pattern: /Baiduspider/i
|
|
546
|
+
},
|
|
547
|
+
{
|
|
548
|
+
id: "yandexadditional",
|
|
549
|
+
name: "YandexAdditional",
|
|
550
|
+
product: "yandex",
|
|
551
|
+
category: "training",
|
|
552
|
+
pattern: /YandexAdditional/i
|
|
553
|
+
},
|
|
554
|
+
{
|
|
555
|
+
id: "yandexbot",
|
|
556
|
+
name: "YandexBot",
|
|
557
|
+
product: "yandex",
|
|
558
|
+
category: "search_engine",
|
|
559
|
+
pattern: /YandexBot|YandexMobileBot/i
|
|
560
|
+
},
|
|
561
|
+
{
|
|
562
|
+
id: "pangubot",
|
|
563
|
+
name: "PanguBot",
|
|
564
|
+
product: "huawei",
|
|
565
|
+
category: "training",
|
|
566
|
+
pattern: /PanguBot/i
|
|
567
|
+
},
|
|
568
|
+
{
|
|
569
|
+
id: "petalbot",
|
|
570
|
+
name: "PetalBot",
|
|
571
|
+
product: "huawei",
|
|
572
|
+
category: "search_engine",
|
|
573
|
+
pattern: /PetalBot/i
|
|
574
|
+
},
|
|
575
|
+
{
|
|
576
|
+
id: "ai2bot",
|
|
577
|
+
name: "AI2Bot",
|
|
578
|
+
product: "ai2",
|
|
579
|
+
category: "training",
|
|
580
|
+
pattern: /AI2Bot|Ai2Bot-Dolma/i
|
|
581
|
+
},
|
|
582
|
+
{
|
|
583
|
+
id: "bravebot",
|
|
584
|
+
name: "Bravebot",
|
|
585
|
+
product: "brave",
|
|
586
|
+
category: "search",
|
|
587
|
+
pattern: /Bravebot/i
|
|
588
|
+
},
|
|
589
|
+
{
|
|
590
|
+
id: "kagi-fetcher",
|
|
591
|
+
name: "Kagi fetcher",
|
|
592
|
+
product: "kagi",
|
|
593
|
+
category: "assistant",
|
|
594
|
+
pattern: /kagi-fetcher/i
|
|
595
|
+
},
|
|
596
|
+
{
|
|
597
|
+
id: "phindbot",
|
|
598
|
+
name: "PhindBot",
|
|
599
|
+
product: "phind",
|
|
600
|
+
category: "assistant",
|
|
601
|
+
pattern: /PhindBot/i
|
|
602
|
+
},
|
|
603
|
+
{
|
|
604
|
+
id: "manus-user",
|
|
605
|
+
name: "Manus",
|
|
606
|
+
product: "manus",
|
|
607
|
+
category: "agent",
|
|
608
|
+
pattern: /Manus-User/i
|
|
609
|
+
},
|
|
610
|
+
{
|
|
611
|
+
id: "cursor",
|
|
612
|
+
name: "Cursor",
|
|
613
|
+
product: "coding",
|
|
614
|
+
category: "assistant",
|
|
615
|
+
pattern: /\bCursor\//i
|
|
616
|
+
},
|
|
617
|
+
{
|
|
618
|
+
id: "devin",
|
|
619
|
+
name: "Devin",
|
|
620
|
+
product: "coding",
|
|
621
|
+
category: "agent",
|
|
622
|
+
pattern: /\bDevin\b/i
|
|
623
|
+
},
|
|
624
|
+
{
|
|
625
|
+
id: "opencode",
|
|
626
|
+
name: "opencode",
|
|
627
|
+
product: "coding",
|
|
628
|
+
category: "assistant",
|
|
629
|
+
pattern: /\bopencode\b/i
|
|
630
|
+
},
|
|
631
|
+
{
|
|
632
|
+
id: "firecrawl",
|
|
633
|
+
name: "Firecrawl",
|
|
634
|
+
product: "aidata",
|
|
635
|
+
category: "assistant",
|
|
636
|
+
pattern: /FirecrawlAgent|Firecrawl/i
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
id: "exabot",
|
|
640
|
+
name: "Exa",
|
|
641
|
+
product: "aidata",
|
|
642
|
+
category: "search",
|
|
643
|
+
pattern: /ExaBot|ExaSearchBot/i
|
|
644
|
+
},
|
|
645
|
+
{
|
|
646
|
+
id: "tavilybot",
|
|
647
|
+
name: "Tavily",
|
|
648
|
+
product: "aidata",
|
|
649
|
+
category: "search",
|
|
650
|
+
pattern: /TavilyBot/i
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
id: "linkupbot",
|
|
654
|
+
name: "Linkup",
|
|
655
|
+
product: "aidata",
|
|
656
|
+
category: "search",
|
|
657
|
+
pattern: /LinkupBot/i
|
|
658
|
+
},
|
|
659
|
+
{
|
|
660
|
+
id: "shapbot",
|
|
661
|
+
name: "Parallel (ShapBot)",
|
|
662
|
+
product: "aidata",
|
|
663
|
+
category: "search",
|
|
664
|
+
pattern: /ShapBot|Shap-User/i
|
|
665
|
+
},
|
|
666
|
+
{
|
|
667
|
+
id: "jina",
|
|
668
|
+
name: "Jina Reader",
|
|
669
|
+
product: "aidata",
|
|
670
|
+
category: "assistant",
|
|
671
|
+
pattern: /Jina-?Reader|\bJina\b/i
|
|
672
|
+
},
|
|
673
|
+
{
|
|
674
|
+
id: "diffbot",
|
|
675
|
+
name: "Diffbot",
|
|
676
|
+
product: "aidata",
|
|
677
|
+
category: "training",
|
|
678
|
+
pattern: /Diffbot/i
|
|
679
|
+
},
|
|
680
|
+
{
|
|
681
|
+
id: "brightbot",
|
|
682
|
+
name: "Bright Data",
|
|
683
|
+
product: "aidata",
|
|
684
|
+
category: "training",
|
|
685
|
+
pattern: /Brightbot/i
|
|
686
|
+
},
|
|
687
|
+
{
|
|
688
|
+
id: "apifybot",
|
|
689
|
+
name: "Apify",
|
|
690
|
+
product: "aidata",
|
|
691
|
+
category: "training",
|
|
692
|
+
pattern: /ApifyBot|ApifyWebsiteContentCrawler/i
|
|
693
|
+
},
|
|
694
|
+
{
|
|
695
|
+
id: "crawl4ai",
|
|
696
|
+
name: "Crawl4AI",
|
|
697
|
+
product: "aidata",
|
|
698
|
+
category: "training",
|
|
699
|
+
pattern: /Crawl4AI/i
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
id: "webzio",
|
|
703
|
+
name: "Webz.io",
|
|
704
|
+
product: "aidata",
|
|
705
|
+
category: "training",
|
|
706
|
+
pattern: /omgili|Webzio-Extended/i
|
|
707
|
+
},
|
|
708
|
+
{
|
|
709
|
+
id: "timpibot",
|
|
710
|
+
name: "Timpibot",
|
|
711
|
+
product: "aidata",
|
|
712
|
+
category: "training",
|
|
713
|
+
pattern: /Timpibot/i
|
|
714
|
+
},
|
|
715
|
+
{
|
|
716
|
+
id: "imagesiftbot",
|
|
717
|
+
name: "ImagesiftBot",
|
|
718
|
+
product: "aidata",
|
|
719
|
+
category: "training",
|
|
720
|
+
pattern: /ImagesiftBot|img2dataset|LAIONDownloader/i
|
|
721
|
+
},
|
|
722
|
+
{
|
|
723
|
+
id: "signed-agent",
|
|
724
|
+
name: "Signed agent",
|
|
725
|
+
product: "automation",
|
|
726
|
+
category: "agent",
|
|
727
|
+
pattern: null
|
|
728
|
+
},
|
|
729
|
+
{
|
|
730
|
+
id: "headless",
|
|
731
|
+
name: "Headless browser",
|
|
732
|
+
product: "automation",
|
|
733
|
+
category: "other",
|
|
734
|
+
pattern: /HeadlessChrome|PhantomJS|Puppeteer|Playwright|Selenium|Lightpanda|webdriver/i
|
|
735
|
+
},
|
|
736
|
+
{
|
|
737
|
+
id: "seo",
|
|
738
|
+
name: "SEO crawler",
|
|
739
|
+
product: "seo",
|
|
740
|
+
category: "other",
|
|
741
|
+
pattern: /AhrefsBot|AhrefsSiteAudit|SemrushBot|SiteAuditBot|MJ12bot|DotBot|rogerbot|Screaming Frog|DataForSeoBot|BLEXBot|serpstatbot|SEOkicks|barkrowler/i
|
|
742
|
+
},
|
|
743
|
+
{
|
|
744
|
+
id: "link-preview",
|
|
745
|
+
name: "Link preview",
|
|
746
|
+
product: "social",
|
|
747
|
+
category: "other",
|
|
748
|
+
pattern: /Twitterbot|LinkedInBot|Slackbot|Slack-ImgProxy|Discordbot|WhatsApp|TelegramBot|Pinterestbot|redditbot|Embedly|Iframely|SkypeUriPreview|vkShare|Mastodon/i
|
|
749
|
+
},
|
|
750
|
+
{
|
|
751
|
+
id: "monitor",
|
|
752
|
+
name: "Uptime monitor",
|
|
753
|
+
product: "monitor",
|
|
754
|
+
category: "other",
|
|
755
|
+
pattern: /UptimeRobot|Pingdom|StatusCake|Better ?Uptime|BetterStack|Checkly|Site24x7|NewRelicPinger|Datadog|UptimeKuma|Uptime-Kuma|Hetrix/i
|
|
756
|
+
},
|
|
757
|
+
{
|
|
758
|
+
id: "search-other",
|
|
759
|
+
name: "Other search engine",
|
|
760
|
+
product: "other",
|
|
761
|
+
category: "search_engine",
|
|
762
|
+
pattern: /Slurp|SeznamBot|Qwantbot|Qwantify|MojeekBot|Ecosia|coccocbot|Sogou|NaverBot|Yeti\//i
|
|
763
|
+
},
|
|
764
|
+
{
|
|
765
|
+
id: "http-client",
|
|
766
|
+
name: "HTTP client / script",
|
|
767
|
+
product: "tools",
|
|
768
|
+
category: "other",
|
|
769
|
+
pattern: /^curl\/|\bWget\b|python-requests|python-urllib|Python\/\d|aiohttp|python-httpx|Go-http-client|okhttp|axios\/|node-fetch|undici|^node$|Java\/\d|Apache-HttpClient|libwww-perl|^Ruby|Faraday|Scrapy|^Deno\/|^Bun\/|GuzzleHttp|^PostmanRuntime|HTTPie/i
|
|
770
|
+
},
|
|
771
|
+
{
|
|
772
|
+
id: "other-bot",
|
|
773
|
+
name: "Other bot",
|
|
774
|
+
product: "other",
|
|
775
|
+
category: "other",
|
|
776
|
+
pattern: /bot\b|bot\/|robot|crawl|spider|scrape|scraping|fetcher|preview|archiver|indexer/i
|
|
777
|
+
},
|
|
778
|
+
{
|
|
779
|
+
id: "no-ua",
|
|
780
|
+
name: "No user agent",
|
|
781
|
+
product: "other",
|
|
782
|
+
category: "other",
|
|
783
|
+
pattern: null
|
|
784
|
+
}
|
|
785
|
+
];
|
|
786
|
+
const BY_ID = new Map(BOT_PATTERNS.map((b) => [b.id, b]));
|
|
787
|
+
/** Real browsers whose UA happens to contain a bot-ish word. */
|
|
788
|
+
const HUMAN_LOOKALIKES = /cubot|\bbotswana\b/i;
|
|
789
|
+
/** Identify a bot from its User-Agent. Returns null for (what looks like) a normal browser. */
|
|
790
|
+
function classifyUserAgent(ua) {
|
|
791
|
+
const s = (ua ?? "").trim();
|
|
792
|
+
if (!s) return BY_ID.get("no-ua");
|
|
793
|
+
for (const b of BOT_PATTERNS) {
|
|
794
|
+
if (!b.pattern || !b.pattern.test(s)) continue;
|
|
795
|
+
if ((b.id === "other-bot" || b.id === "link-preview") && HUMAN_LOOKALIKES.test(s)) continue;
|
|
796
|
+
return b;
|
|
797
|
+
}
|
|
798
|
+
return null;
|
|
799
|
+
}
|
|
800
|
+
/** Hostnames of AI assistants that send people to websites, and the `utm_source` values they add. */
|
|
801
|
+
const AI_REFERRERS = [
|
|
802
|
+
{
|
|
803
|
+
product: "chatgpt",
|
|
804
|
+
hosts: /(^|\.)(chatgpt\.com|chat\.openai\.com)$/,
|
|
805
|
+
utm: /^(chatgpt(\.com)?|openai)$/
|
|
806
|
+
},
|
|
807
|
+
{
|
|
808
|
+
product: "perplexity",
|
|
809
|
+
hosts: /(^|\.)perplexity\.ai$/,
|
|
810
|
+
utm: /^perplexity(\.ai)?$/
|
|
811
|
+
},
|
|
812
|
+
{
|
|
813
|
+
product: "claude",
|
|
814
|
+
hosts: /(^|\.)claude\.ai$/,
|
|
815
|
+
utm: /^claude(\.ai)?$/
|
|
816
|
+
},
|
|
817
|
+
{
|
|
818
|
+
product: "gemini",
|
|
819
|
+
hosts: /^(gemini|bard)\.google\.com$/,
|
|
820
|
+
utm: /^(gemini|bard)$/
|
|
821
|
+
},
|
|
822
|
+
{
|
|
823
|
+
product: "copilot",
|
|
824
|
+
hosts: /^copilot\.microsoft\.com$|(^|\.)copilot\.cloud\.microsoft$|^copilot\.com$/,
|
|
825
|
+
utm: /^copilot(\.com)?$/
|
|
826
|
+
},
|
|
827
|
+
{
|
|
828
|
+
product: "meta",
|
|
829
|
+
hosts: /(^|\.)meta\.ai$/,
|
|
830
|
+
utm: /^meta\.?ai$/
|
|
831
|
+
},
|
|
832
|
+
{
|
|
833
|
+
product: "deepseek",
|
|
834
|
+
hosts: /(^|\.)deepseek\.com$/,
|
|
835
|
+
utm: /^deepseek$/
|
|
836
|
+
},
|
|
837
|
+
{
|
|
838
|
+
product: "xai",
|
|
839
|
+
hosts: /(^|\.)grok\.com$/,
|
|
840
|
+
utm: /^grok$/
|
|
841
|
+
},
|
|
842
|
+
{
|
|
843
|
+
product: "mistral",
|
|
844
|
+
hosts: /^chat\.mistral\.ai$/,
|
|
845
|
+
utm: /^(le-?chat|mistral)$/
|
|
846
|
+
},
|
|
847
|
+
{
|
|
848
|
+
product: "you",
|
|
849
|
+
hosts: /(^|\.)you\.com$/
|
|
850
|
+
},
|
|
851
|
+
{
|
|
852
|
+
product: "phind",
|
|
853
|
+
hosts: /(^|\.)phind\.com$/
|
|
854
|
+
},
|
|
855
|
+
{
|
|
856
|
+
product: "poe",
|
|
857
|
+
hosts: /(^|\.)poe\.com$/
|
|
858
|
+
},
|
|
859
|
+
{
|
|
860
|
+
product: "duckduckgo",
|
|
861
|
+
hosts: /^duck\.ai$/
|
|
862
|
+
},
|
|
863
|
+
{
|
|
864
|
+
product: "qwen",
|
|
865
|
+
hosts: /^chat\.qwen\.ai$/
|
|
866
|
+
},
|
|
867
|
+
{
|
|
868
|
+
product: "kimi",
|
|
869
|
+
hosts: /(^|\.)kimi\.(com|ai)$|^kimi\.moonshot\.cn$/
|
|
870
|
+
},
|
|
871
|
+
{
|
|
872
|
+
product: "bytedance",
|
|
873
|
+
hosts: /(^|\.)doubao\.com$/
|
|
874
|
+
}
|
|
875
|
+
];
|
|
876
|
+
function hostOf(raw) {
|
|
877
|
+
try {
|
|
878
|
+
return new URL(raw.includes("://") ? raw : `https://${raw}`).hostname.toLowerCase().replace(/^www\./, "");
|
|
879
|
+
} catch {
|
|
880
|
+
return "";
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
/**
|
|
884
|
+
* Which AI assistant sent a human to this page, from the Referer header and/or
|
|
885
|
+
* the landing URL's `utm_source`. Returns the product id, or null.
|
|
886
|
+
*/
|
|
887
|
+
function classifyReferral(referrer, url) {
|
|
888
|
+
if (referrer) {
|
|
889
|
+
const host = hostOf(referrer);
|
|
890
|
+
const hit = host && AI_REFERRERS.find((r) => r.hosts.test(host));
|
|
891
|
+
if (hit) return hit.product;
|
|
892
|
+
}
|
|
893
|
+
if (url && url.includes("utm_source=")) {
|
|
894
|
+
const m = url.match(/[?&]utm_source=([^&#]+)/);
|
|
895
|
+
const src = m ? decodeURIComponent(m[1]).toLowerCase().replace(/^www\./, "") : "";
|
|
896
|
+
const hit = src && AI_REFERRERS.find((r) => r.utm?.test(src) || r.hosts.test(src));
|
|
897
|
+
if (hit) return hit.product;
|
|
898
|
+
}
|
|
899
|
+
return null;
|
|
900
|
+
}
|
|
901
|
+
function readHeader(h, name) {
|
|
902
|
+
if (!h) return null;
|
|
903
|
+
if (typeof h.get === "function") return h.get(name);
|
|
904
|
+
const v = h[name];
|
|
905
|
+
return Array.isArray(v) ? v.join(", ") : v ?? null;
|
|
906
|
+
}
|
|
907
|
+
/**
|
|
908
|
+
* What kind of client sent this request, from its User-Agent, Web Bot Auth headers and referrer.
|
|
909
|
+
* Synchronous and dependency-free, so it works in any server, middleware or edge function.
|
|
910
|
+
*
|
|
911
|
+
* It reports who the client *claims* to be. Anyone can send `User-Agent: GPTBot`, so don't
|
|
912
|
+
* use it to grant access; your dashboard verifies each hit against the vendor's IP ranges,
|
|
913
|
+
* reverse DNS and signatures.
|
|
914
|
+
*
|
|
915
|
+
* detectBot(request) // fetch Request (Next, SvelteKit, Astro, Hono, Workers…)
|
|
916
|
+
* detectBot(await headers()) // Next.js server components
|
|
917
|
+
* detectBot(req) // Node / Express IncomingMessage
|
|
918
|
+
*/
|
|
919
|
+
function detectBot(input) {
|
|
920
|
+
const headers = input && typeof input.get === "function" ? input : input?.headers;
|
|
921
|
+
return detectFromHeaders((name) => readHeader(headers, name), input && "url" in input && typeof input.url === "string" ? input.url : null);
|
|
922
|
+
}
|
|
923
|
+
/** `detectBot` over a header getter (used by the server adapters). */
|
|
924
|
+
function detectFromHeaders(header, url) {
|
|
925
|
+
const ua = header("user-agent") ?? null;
|
|
926
|
+
const signed = !!(header("signature-agent") && header("signature-input"));
|
|
927
|
+
const bot = classifyUserAgent(ua) ?? (signed ? BY_ID.get("signed-agent") : null);
|
|
928
|
+
if (!bot) {
|
|
929
|
+
const referral = classifyReferral(header("referer"), url);
|
|
930
|
+
return {
|
|
931
|
+
isBot: false,
|
|
932
|
+
isAI: false,
|
|
933
|
+
id: null,
|
|
934
|
+
name: null,
|
|
935
|
+
category: null,
|
|
936
|
+
product: null,
|
|
937
|
+
operator: null,
|
|
938
|
+
browser: false,
|
|
939
|
+
signed: false,
|
|
940
|
+
referral: referral ? BOT_PRODUCTS[referral].name : null
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
const product = BOT_PRODUCTS[bot.product];
|
|
944
|
+
return {
|
|
945
|
+
isBot: true,
|
|
946
|
+
isAI: isAiCategory(bot.category),
|
|
947
|
+
id: bot.id,
|
|
948
|
+
name: bot.name,
|
|
949
|
+
category: bot.category,
|
|
950
|
+
product: product.name,
|
|
951
|
+
operator: product.operator,
|
|
952
|
+
browser: bot.category === "agent" || bot.id === "headless",
|
|
953
|
+
signed,
|
|
954
|
+
referral: null
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
//#endregion
|
|
958
|
+
//#region src/server.ts
|
|
959
|
+
/**
|
|
960
|
+
* AgentStats server integration — see which AI crawlers and agents visit your
|
|
961
|
+
* site (GPTBot, ClaudeBot, PerplexityBot, ChatGPT agent, Google-Agent, …),
|
|
962
|
+
* including the ones that never run JavaScript.
|
|
963
|
+
*
|
|
964
|
+
* Runs on Node and on edge runtimes (no Node built-ins). What leaves your server:
|
|
965
|
+
* - bot-like requests: method, URL, status, user agent, IP (used to verify the
|
|
966
|
+
* bot against the vendor's published ranges, then discarded) and Web Bot Auth
|
|
967
|
+
* signature headers;
|
|
968
|
+
* - visits referred by an AI assistant (chatgpt.com, perplexity.ai, …): URL and
|
|
969
|
+
* referrer host only;
|
|
970
|
+
* - everything else: a per-day count of page views. Nothing about humans.
|
|
971
|
+
*
|
|
972
|
+
* import { createAgentStats } from '@agentstats/sdk/server'
|
|
973
|
+
* const lens = createAgentStats({ serverKey: process.env.AGENTSTATS_SERVER_KEY })
|
|
974
|
+
* app.use(lens.express()) // Express / Connect: res.locals.bot
|
|
975
|
+
* export default defineNitroPlugin(lens.nitro()) // Nuxt / Nitro: event.context.bot
|
|
976
|
+
* export const handle = lens.sveltekit() // SvelteKit: event.locals.bot
|
|
977
|
+
* export const onRequest = lens.astro() // Astro: Astro.locals.bot
|
|
978
|
+
* app.use(lens.hono()) // Hono: c.get('bot')
|
|
979
|
+
* lens.track(request, { waitUntil, response }) // Next.js proxy, Workers, Bun, Deno
|
|
980
|
+
*
|
|
981
|
+
* Every adapter also hands you `detectBot()`'s answer for the request, and `detectBot`
|
|
982
|
+
* works on its own (no key needed): `detectBot(request).isAI`.
|
|
983
|
+
*
|
|
984
|
+
* With an `apiKey` (an API's key, `al_api_…`), the same adapters also track your API: calls an
|
|
985
|
+
* AI agent made (a Web Bot Auth signature, an `X-AI-Agent` header, an agent's User-Agent, or a
|
|
986
|
+
* credential you name in `apiCaller`) with method, route pattern, status and duration, and a
|
|
987
|
+
* per-day count of every other call. Never bodies, query strings or credentials: the caller is
|
|
988
|
+
* sent as an HMAC of its credential, keyed with your API key.
|
|
989
|
+
*
|
|
990
|
+
* const lens = createAgentStats({ apiKey: process.env.AGENTSTATS_API_KEY }) // API-only server
|
|
991
|
+
*/
|
|
992
|
+
const SERVER_SDK_VERSION = "0.3.1";
|
|
993
|
+
/**
|
|
994
|
+
* Words that may mean a bot the table below doesn't know yet. Deliberately broad: the server
|
|
995
|
+
* makes the final call, and its table can be newer than this SDK's.
|
|
996
|
+
*/
|
|
997
|
+
const BOT_HINT = /bot|crawl|spider|slurp|scrap|fetch|agent|preview|headless|python|curl|wget|httpx|http-?client|go-http|java\/|okhttp|axios|node|undici|libwww|perl|ruby|phantom|puppeteer|playwright|selenium|lighthouse|monitor|uptime|gpt|openai|claude|anthropic|perplexity|gemini|google-|googleother|meta-external|facebookexternalhit|bytespider|ccbot|diffbot|cohere|mistral|deepseek|manus|firecrawl|jina|tavily|exabot|linkup|novaact|amzn|cursor|devin|opencode|kagi|duckassist|qwen|tongyi|kimi|yiyan|ernie|doubao|petal|pangu|timpi|omgili|webzio|imagesift|img2dataset|lightpanda|crawl4ai|apify|youbot|phind/i;
|
|
998
|
+
const ASSET = /\.(?:js|mjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|bmp|woff2?|ttf|otf|eot|mp4|webm|mov|mp3|wav|ogg|zip|gz|wasm|pdf)$/i;
|
|
999
|
+
const SIGNATURE_HEADERS = [
|
|
1000
|
+
"signature",
|
|
1001
|
+
"signature-input",
|
|
1002
|
+
"signature-agent"
|
|
1003
|
+
];
|
|
1004
|
+
const NEVER_FORWARD = /* @__PURE__ */ new Set([
|
|
1005
|
+
"cookie",
|
|
1006
|
+
"authorization",
|
|
1007
|
+
"proxy-authorization",
|
|
1008
|
+
"x-api-key"
|
|
1009
|
+
]);
|
|
1010
|
+
const VISIT_COOKIE = "_al_v";
|
|
1011
|
+
/** Paths that are API calls when the server also serves a website. */
|
|
1012
|
+
const API_PATH = /^\/(?:api|v\d+)(?:\/|$)/;
|
|
1013
|
+
/** A path segment that's a value rather than part of the route: numbers, UUIDs, hashes, long tokens, emails. */
|
|
1014
|
+
const ID_SEGMENT = /^(\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{12,}|(?=[^/]*\d)[\w-]{16,}|[^/@]+@[^/]+|[A-Za-z]+_[A-Za-z0-9]{12,})$/i;
|
|
1015
|
+
/** A path as a route pattern: values (ids, UUIDs, tokens, emails) replaced by `:id`. */
|
|
1016
|
+
function routeFromPath(path) {
|
|
1017
|
+
return path.split("/").map((s) => {
|
|
1018
|
+
let v = s;
|
|
1019
|
+
try {
|
|
1020
|
+
v = decodeURIComponent(s);
|
|
1021
|
+
} catch {}
|
|
1022
|
+
return s && ID_SEGMENT.test(v) ? ":id" : s;
|
|
1023
|
+
}).join("/") || "/";
|
|
1024
|
+
}
|
|
1025
|
+
/** A framework route pattern as `/users/:id`: SvelteKit and Astro `[id]`/`[...rest]`, minus route groups. */
|
|
1026
|
+
function normalizeRoutePattern(pattern) {
|
|
1027
|
+
return pattern.replace(/\/\([^/)]+\)(?=\/|$)/g, "").replace(/\[\[?\.\.\.([^\]]+)\]\]?/g, ":$1*").replace(/\[\[?([^\]]+)\]\]?/g, ":$1") || "/";
|
|
1028
|
+
}
|
|
1029
|
+
/** The framework's route pattern, if it fits the request's path segment for segment (else it's a parent router's or a middleware's). */
|
|
1030
|
+
function fitRoute(pattern, path) {
|
|
1031
|
+
if (typeof pattern !== "string" || !pattern.startsWith("/")) return null;
|
|
1032
|
+
const p = normalizeRoutePattern(pattern).replace(/\/$/, "") || "/";
|
|
1033
|
+
if (p.includes("*")) return p;
|
|
1034
|
+
const segments = (s) => s.split("/").filter(Boolean).length;
|
|
1035
|
+
return segments(p) === segments(path) ? p : null;
|
|
1036
|
+
}
|
|
1037
|
+
/** Web Bot Auth headers and the headers their signature covers, so the server can verify it. Never credentials or cookies. */
|
|
1038
|
+
function signatureHeaders(header) {
|
|
1039
|
+
const headers = {};
|
|
1040
|
+
const covered = (header("signature-input") ?? "").match(/"([a-z0-9-]+)"/gi)?.map((s) => s.slice(1, -1).toLowerCase()) ?? [];
|
|
1041
|
+
for (const name of [...SIGNATURE_HEADERS, ...covered].slice(0, 16)) {
|
|
1042
|
+
if (name.startsWith("@") || NEVER_FORWARD.has(name)) continue;
|
|
1043
|
+
const v = header(name);
|
|
1044
|
+
if (v) headers[name] = v;
|
|
1045
|
+
}
|
|
1046
|
+
return headers;
|
|
1047
|
+
}
|
|
1048
|
+
function defaultCaller(header) {
|
|
1049
|
+
return header("authorization")?.trim() || header("x-api-key")?.trim() || header("api-key")?.trim() || null;
|
|
1050
|
+
}
|
|
1051
|
+
function defaultEndpoint() {
|
|
1052
|
+
const env = globalThis.process?.env;
|
|
1053
|
+
if (env?.AGENTSTATS_ENDPOINT) return env.AGENTSTATS_ENDPOINT;
|
|
1054
|
+
return "https://agentstats.io/api/ingest".replace(/\/api\/ingest\/?$/, "/api/ingest/hits");
|
|
1055
|
+
}
|
|
1056
|
+
function randomId() {
|
|
1057
|
+
const c = globalThis.crypto;
|
|
1058
|
+
if (c?.randomUUID) return c.randomUUID().replace(/-/g, "");
|
|
1059
|
+
return Array.from({ length: 32 }, () => Math.floor(Math.random() * 16).toString(16)).join("");
|
|
1060
|
+
}
|
|
1061
|
+
function firstIp(v) {
|
|
1062
|
+
return v?.split(",")[0]?.trim() || null;
|
|
1063
|
+
}
|
|
1064
|
+
/** The client-IP header of the platform we run on. These platforms overwrite it, so a client can't forge it. */
|
|
1065
|
+
function platformIpHeader() {
|
|
1066
|
+
const g = globalThis;
|
|
1067
|
+
const env = g.process?.env ?? {};
|
|
1068
|
+
if (g.navigator?.userAgent === "Cloudflare-Workers") return "cf-connecting-ip";
|
|
1069
|
+
if (env.VERCEL) return "x-real-ip";
|
|
1070
|
+
if (g.Netlify || env.NETLIFY) return "x-nf-client-connection-ip";
|
|
1071
|
+
if (env.FLY_APP_NAME) return "fly-client-ip";
|
|
1072
|
+
return null;
|
|
1073
|
+
}
|
|
1074
|
+
/** Forwarding headers in order of preference, for proxies we can't identify. */
|
|
1075
|
+
function forwardedIp(h) {
|
|
1076
|
+
return firstIp(h("cf-connecting-ip") ?? h("true-client-ip") ?? h("x-real-ip") ?? h("x-vercel-forwarded-for") ?? h("fly-client-ip") ?? h("x-forwarded-for"));
|
|
1077
|
+
}
|
|
1078
|
+
/** `waitUntil` of the current request on platforms that expose it globally (Vercel functions, Netlify). */
|
|
1079
|
+
function ambientWaitUntil() {
|
|
1080
|
+
const g = globalThis;
|
|
1081
|
+
const vercel = g[Symbol.for("@vercel/request-context")]?.get?.();
|
|
1082
|
+
if (typeof vercel?.waitUntil === "function") return (p) => vercel.waitUntil(p);
|
|
1083
|
+
const netlify = g.Netlify?.context;
|
|
1084
|
+
if (typeof netlify?.waitUntil === "function") return (p) => netlify.waitUntil(p);
|
|
1085
|
+
}
|
|
1086
|
+
/** Loopback, private (RFC 1918 / ULA), link-local and carrier-grade NAT addresses: never a crawler's real IP. */
|
|
1087
|
+
function isPrivateIp(ip) {
|
|
1088
|
+
if (!ip) return false;
|
|
1089
|
+
const s = ip.toLowerCase().replace(/^\[|\]$/g, "").replace(/%.*$/, "").replace(/^::ffff:(?=\d+\.\d+\.\d+\.\d+$)/, "");
|
|
1090
|
+
const v4 = s.match(/^(\d+)\.(\d+)\.\d+\.\d+$/);
|
|
1091
|
+
if (v4) {
|
|
1092
|
+
const a = Number(v4[1]);
|
|
1093
|
+
const b = Number(v4[2]);
|
|
1094
|
+
return a === 10 || a === 127 || a === 0 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 169 && b === 254 || a === 100 && b >= 64 && b <= 127;
|
|
1095
|
+
}
|
|
1096
|
+
return s === "::1" || s === "::" || /^f[cd][0-9a-f]{0,2}:/.test(s) || /^fe[89ab][0-9a-f]?:/.test(s);
|
|
1097
|
+
}
|
|
1098
|
+
let dayCache = {
|
|
1099
|
+
from: 0,
|
|
1100
|
+
to: 0,
|
|
1101
|
+
value: ""
|
|
1102
|
+
};
|
|
1103
|
+
function day(ts) {
|
|
1104
|
+
if (ts < dayCache.from || ts >= dayCache.to) {
|
|
1105
|
+
const from = ts - ts % 864e5;
|
|
1106
|
+
dayCache = {
|
|
1107
|
+
from,
|
|
1108
|
+
to: from + 864e5,
|
|
1109
|
+
value: new Date(from).toISOString().slice(0, 10)
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
return dayCache.value;
|
|
1113
|
+
}
|
|
1114
|
+
const now = () => globalThis.performance?.now() ?? Date.now();
|
|
1115
|
+
var AgentStatsServer = class {
|
|
1116
|
+
constructor(opts) {
|
|
1117
|
+
this.queue = [];
|
|
1118
|
+
this.humans = {};
|
|
1119
|
+
this.humanCount = 0;
|
|
1120
|
+
this.apiQueue = [];
|
|
1121
|
+
this.apiOther = {};
|
|
1122
|
+
this.apiOtherCount = 0;
|
|
1123
|
+
this.apiHmacKey = null;
|
|
1124
|
+
this.apiHmacs = /* @__PURE__ */ new Map();
|
|
1125
|
+
this.timer = null;
|
|
1126
|
+
this.lastFlush = 0;
|
|
1127
|
+
this.inflight = Promise.resolve();
|
|
1128
|
+
if (!opts?.serverKey && !opts?.apiKey && !opts?.disabled) console.warn("[agentstats] no `serverKey`: detecting bots only, nothing is sent to AgentStats (Site settings → Server key)");
|
|
1129
|
+
if (opts.serverKey && !opts.serverKey.startsWith("al_srv_")) throw new Error("[agentstats] `serverKey` must be the secret server key (al_srv_…), not the public site key");
|
|
1130
|
+
if (opts.apiKey && !opts.apiKey.startsWith("al_api_")) throw new Error("[agentstats] `apiKey` must be an API's key (al_api_…, Project settings → APIs)");
|
|
1131
|
+
const endpoint = opts?.endpoint ?? defaultEndpoint();
|
|
1132
|
+
if (!endpoint && (opts?.serverKey || opts?.apiKey) && !opts?.disabled) throw new Error("[agentstats] `endpoint` is required, e.g. https://your-host/api/ingest/hits");
|
|
1133
|
+
this.options = {
|
|
1134
|
+
countHumans: true,
|
|
1135
|
+
visitCookie: true,
|
|
1136
|
+
maxBatch: 100,
|
|
1137
|
+
flushIntervalMs: 5e3,
|
|
1138
|
+
debug: false,
|
|
1139
|
+
disabled: false,
|
|
1140
|
+
...Object.fromEntries(Object.entries(opts ?? {}).filter(([, v]) => v !== void 0)),
|
|
1141
|
+
endpoint,
|
|
1142
|
+
apiEndpoint: opts?.apiEndpoint ?? endpoint.replace(/\/hits\/?$/, "/api")
|
|
1143
|
+
};
|
|
1144
|
+
this.hitsOn = !!opts?.serverKey && !this.options.disabled;
|
|
1145
|
+
this.apiOn = !!opts?.apiKey && !this.options.disabled;
|
|
1146
|
+
this._fetch = opts?.fetch ?? ((...a) => fetch(...a));
|
|
1147
|
+
}
|
|
1148
|
+
/** Express / Connect middleware: `app.use(lens.express())`. The detection is on `res.locals.bot`. */
|
|
1149
|
+
express() {
|
|
1150
|
+
return (req, res, next) => {
|
|
1151
|
+
const bot = this.node(req, res);
|
|
1152
|
+
if (res.locals) res.locals.bot = bot;
|
|
1153
|
+
next?.();
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
/** Plain Node `http`. Call before the response is written. Returns what `detectBot` says about the request. */
|
|
1157
|
+
node(req, res) {
|
|
1158
|
+
const get = (name) => headerValue(req.headers[name]);
|
|
1159
|
+
let d;
|
|
1160
|
+
try {
|
|
1161
|
+
d = this.decide(req.method ?? "GET", this.nodeUrl(req), get, this.nodeIp(req));
|
|
1162
|
+
if (d.kind === "ignore") return d.detection;
|
|
1163
|
+
if (d.cookie) appendNodeCookie(res, d.cookie);
|
|
1164
|
+
const decided = d;
|
|
1165
|
+
res.once?.("finish", () => {
|
|
1166
|
+
if (typeof req.route?.path === "string") this.setRoute(decided, (req.baseUrl ?? "") + req.route.path);
|
|
1167
|
+
this.setContentType(decided, res.getHeader?.("content-type"));
|
|
1168
|
+
this.commit(decided, res.statusCode ?? null);
|
|
1169
|
+
});
|
|
1170
|
+
if (!res.once) this.commit(d, null);
|
|
1171
|
+
} catch (err) {
|
|
1172
|
+
this.log("track failed", err);
|
|
1173
|
+
}
|
|
1174
|
+
return d?.detection ?? detectFromHeaders(get, req.url ?? null);
|
|
1175
|
+
}
|
|
1176
|
+
/**
|
|
1177
|
+
* Fetch-API requests (Next.js proxy/middleware, Cloudflare Workers, Bun, Deno).
|
|
1178
|
+
* Pass `response` to let AgentStats set the visit cookie on it, and `waitUntil` on serverless/edge
|
|
1179
|
+
* so the hit is sent after the response. Returns what `detectBot` says about the request.
|
|
1180
|
+
*/
|
|
1181
|
+
track(request, init = {}) {
|
|
1182
|
+
const d = this.fetchDecision(request, init.ip);
|
|
1183
|
+
this.setRoute(d, init.route);
|
|
1184
|
+
this.fetchFinish(d, init.status ?? null, init.response, init.waitUntil);
|
|
1185
|
+
return d.detection;
|
|
1186
|
+
}
|
|
1187
|
+
/** Hono middleware: `app.use(lens.hono())`. The detection is on `c.get('bot')`. */
|
|
1188
|
+
hono() {
|
|
1189
|
+
return async (c, next) => {
|
|
1190
|
+
let ip;
|
|
1191
|
+
try {
|
|
1192
|
+
const incoming = c.env?.incoming;
|
|
1193
|
+
if (incoming?.socket) ip = this.nodeIp(incoming);
|
|
1194
|
+
else if (typeof c.env?.requestIP === "function") ip = this.options.ipHeader || this.options.trustProxy || platformIpHeader() ? void 0 : c.env.requestIP(c.req.raw)?.address;
|
|
1195
|
+
} catch {}
|
|
1196
|
+
const d = this.fetchDecision(c.req.raw, ip);
|
|
1197
|
+
c.set?.("bot", d.detection);
|
|
1198
|
+
await next();
|
|
1199
|
+
try {
|
|
1200
|
+
const routes = c.req.matchedRoutes?.filter((r) => r.method !== "ALL") ?? [];
|
|
1201
|
+
this.setRoute(d, routes[routes.length - 1]?.path ?? c.req.routePath);
|
|
1202
|
+
} catch {}
|
|
1203
|
+
let waitUntil;
|
|
1204
|
+
try {
|
|
1205
|
+
const ctx = c.executionCtx;
|
|
1206
|
+
if (ctx?.waitUntil) waitUntil = ctx.waitUntil.bind(ctx);
|
|
1207
|
+
} catch {}
|
|
1208
|
+
this.fetchFinish(d, c.res?.status ?? null, c.res, waitUntil);
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
/** SvelteKit: `export const handle = lens.sveltekit()` in `src/hooks.server.ts` (or inside `sequence()`). The detection is on `event.locals.bot`. */
|
|
1212
|
+
sveltekit() {
|
|
1213
|
+
return async ({ event, resolve }) => {
|
|
1214
|
+
let ip;
|
|
1215
|
+
try {
|
|
1216
|
+
ip = event.getClientAddress();
|
|
1217
|
+
} catch {
|
|
1218
|
+
event.locals.bot = detectBot(event.request);
|
|
1219
|
+
return resolve(event);
|
|
1220
|
+
}
|
|
1221
|
+
const d = event.isSubRequest ? null : this.fetchDecision(event.request, ip);
|
|
1222
|
+
event.locals.bot = d?.detection ?? detectBot(event.request);
|
|
1223
|
+
const res = await resolve(event);
|
|
1224
|
+
if (d) {
|
|
1225
|
+
this.setRoute(d, event.route?.id);
|
|
1226
|
+
const p = event.platform;
|
|
1227
|
+
const ctx = p?.context?.waitUntil ? p.context : p?.ctx;
|
|
1228
|
+
this.fetchFinish(d, res.status, res, ctx?.waitUntil?.bind(ctx));
|
|
1229
|
+
}
|
|
1230
|
+
return res;
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
/** Astro: `export const onRequest = lens.astro()` in `src/middleware.ts` (or inside `sequence()`). The detection is on `Astro.locals.bot`. */
|
|
1234
|
+
astro() {
|
|
1235
|
+
return async (context, next) => {
|
|
1236
|
+
if (context.isPrerendered) return next();
|
|
1237
|
+
let ip = null;
|
|
1238
|
+
try {
|
|
1239
|
+
ip = context.clientAddress ?? null;
|
|
1240
|
+
} catch {}
|
|
1241
|
+
const d = this.fetchDecision(context.request, ip);
|
|
1242
|
+
const locals = context.locals;
|
|
1243
|
+
locals.bot = d.detection;
|
|
1244
|
+
const res = await next();
|
|
1245
|
+
this.setRoute(d, context.routePattern);
|
|
1246
|
+
const ctx = locals.runtime?.ctx;
|
|
1247
|
+
this.fetchFinish(d, res.status, res, ctx?.waitUntil?.bind(ctx));
|
|
1248
|
+
return res;
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
/**
|
|
1252
|
+
* Nuxt / Nitro plugin: `export default defineNitroPlugin(lens.nitro())` in `server/plugins/agentstats.ts`.
|
|
1253
|
+
* The detection is on `event.context.bot` in server routes, and on `useRequestEvent()?.context.bot` in pages.
|
|
1254
|
+
* Works on every Nitro preset (Node, Vercel, Netlify, Cloudflare…).
|
|
1255
|
+
*/
|
|
1256
|
+
nitro() {
|
|
1257
|
+
return (nitroApp) => {
|
|
1258
|
+
nitroApp.hooks.hook("request", (event) => {
|
|
1259
|
+
const d = this.h3Decision(event);
|
|
1260
|
+
event.context.bot = d.detection;
|
|
1261
|
+
if (d.kind === "ignore") return;
|
|
1262
|
+
event.context[DECISION] = d;
|
|
1263
|
+
if (d.cookie && event.node?.res) appendNodeCookie(event.node.res, d.cookie);
|
|
1264
|
+
});
|
|
1265
|
+
const finish = (event, status) => {
|
|
1266
|
+
const d = event?.context[DECISION];
|
|
1267
|
+
if (!d) return;
|
|
1268
|
+
delete event.context[DECISION];
|
|
1269
|
+
this.setRoute(d, event.context.matchedRoute?.path);
|
|
1270
|
+
this.setContentType(d, event.node?.res?.getHeader?.("content-type"));
|
|
1271
|
+
this.commit(d, status, typeof event.waitUntil === "function" ? event.waitUntil.bind(event) : void 0);
|
|
1272
|
+
};
|
|
1273
|
+
nitroApp.hooks.hook("afterResponse", (event) => finish(event, event.node?.res?.statusCode ?? null));
|
|
1274
|
+
nitroApp.hooks.hook("error", (error, context = {}) => finish(context.event, error?.statusCode ?? 500));
|
|
1275
|
+
};
|
|
1276
|
+
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Wrap a `fetch(request, env, ctx)` handler (Cloudflare Workers, Bun.serve, Deno.serve), or a Next.js
|
|
1279
|
+
* route handler. `route` names the handler's route pattern for API tracking, e.g. `/api/users/[id]`.
|
|
1280
|
+
*/
|
|
1281
|
+
wrapFetch(handler, opts = {}) {
|
|
1282
|
+
return async (request, ...rest) => {
|
|
1283
|
+
const d = this.fetchDecision(request);
|
|
1284
|
+
this.setRoute(d, opts.route);
|
|
1285
|
+
const res = await handler(request, ...rest);
|
|
1286
|
+
const ctx = rest[1];
|
|
1287
|
+
let out = res;
|
|
1288
|
+
try {
|
|
1289
|
+
out = new Response(res.body, res);
|
|
1290
|
+
} catch {}
|
|
1291
|
+
this.fetchFinish(d, res.status, out, ctx?.waitUntil?.bind(ctx));
|
|
1292
|
+
return out;
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
/** Send everything queued now. */
|
|
1296
|
+
flush() {
|
|
1297
|
+
if (this.timer) {
|
|
1298
|
+
clearTimeout(this.timer);
|
|
1299
|
+
this.timer = null;
|
|
1300
|
+
}
|
|
1301
|
+
const sends = [];
|
|
1302
|
+
if (this.queue.length || this.humanCount) sends.push(this.flushHits());
|
|
1303
|
+
if (this.apiQueue.length || this.apiOtherCount) sends.push(this.flushApi());
|
|
1304
|
+
if (!sends.length) return this.inflight;
|
|
1305
|
+
this.lastFlush = Date.now();
|
|
1306
|
+
const p = Promise.all(sends).then(() => {});
|
|
1307
|
+
this.inflight = p;
|
|
1308
|
+
return p;
|
|
1309
|
+
}
|
|
1310
|
+
flushHits() {
|
|
1311
|
+
const hits = this.queue.splice(0, this.queue.length);
|
|
1312
|
+
const humans = this.humans;
|
|
1313
|
+
this.humans = {};
|
|
1314
|
+
this.humanCount = 0;
|
|
1315
|
+
const body = JSON.stringify({
|
|
1316
|
+
sdk: `node/${SERVER_SDK_VERSION}`,
|
|
1317
|
+
humans,
|
|
1318
|
+
hits
|
|
1319
|
+
});
|
|
1320
|
+
return this._fetch(this.options.endpoint, {
|
|
1321
|
+
method: "POST",
|
|
1322
|
+
body,
|
|
1323
|
+
headers: {
|
|
1324
|
+
"content-type": "application/json",
|
|
1325
|
+
"authorization": `Bearer ${this.options.serverKey}`
|
|
1326
|
+
}
|
|
1327
|
+
}).then((res) => {
|
|
1328
|
+
if (!res.ok) this.log("ingest responded", res.status);
|
|
1329
|
+
}).catch((err) => {
|
|
1330
|
+
this.log("ingest failed", err);
|
|
1331
|
+
if (this.queue.length < 1e3) this.queue.unshift(...hits.slice(0, 1e3 - this.queue.length));
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
async flushApi() {
|
|
1335
|
+
const items = this.apiQueue.splice(0, this.apiQueue.length);
|
|
1336
|
+
const other = this.apiOther;
|
|
1337
|
+
this.apiOther = {};
|
|
1338
|
+
this.apiOtherCount = 0;
|
|
1339
|
+
for (const item of items) {
|
|
1340
|
+
if (item.caller !== null) item.record.actor = await this.hmac(item.caller);
|
|
1341
|
+
item.caller = null;
|
|
1342
|
+
}
|
|
1343
|
+
const body = JSON.stringify({
|
|
1344
|
+
calls: items.map((i) => i.record),
|
|
1345
|
+
other
|
|
1346
|
+
});
|
|
1347
|
+
return this._fetch(this.options.apiEndpoint, {
|
|
1348
|
+
method: "POST",
|
|
1349
|
+
body,
|
|
1350
|
+
headers: {
|
|
1351
|
+
"content-type": "application/json",
|
|
1352
|
+
"authorization": `Bearer ${this.options.apiKey}`,
|
|
1353
|
+
"x-agentstats-sdk": `node/${SERVER_SDK_VERSION}`
|
|
1354
|
+
}
|
|
1355
|
+
}).then((res) => {
|
|
1356
|
+
if (!res.ok) this.log("api ingest responded", res.status);
|
|
1357
|
+
}).catch((err) => {
|
|
1358
|
+
this.log("api ingest failed", err);
|
|
1359
|
+
if (this.apiQueue.length < 1e3) this.apiQueue.unshift(...items.slice(0, 1e3 - this.apiQueue.length));
|
|
1360
|
+
for (const [k, n] of Object.entries(other)) {
|
|
1361
|
+
this.apiOther[k] = (this.apiOther[k] ?? 0) + n;
|
|
1362
|
+
this.apiOtherCount += n;
|
|
1363
|
+
}
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
/** HMAC-SHA256 of a credential keyed with the API key (hex, 128 bits), or null without Web Crypto. */
|
|
1367
|
+
hmac(value) {
|
|
1368
|
+
let h = this.apiHmacs.get(value);
|
|
1369
|
+
if (!h) {
|
|
1370
|
+
if (this.apiHmacs.size >= 1e3) this.apiHmacs.delete(this.apiHmacs.keys().next().value);
|
|
1371
|
+
h = this.computeHmac(value);
|
|
1372
|
+
this.apiHmacs.set(value, h);
|
|
1373
|
+
}
|
|
1374
|
+
return h;
|
|
1375
|
+
}
|
|
1376
|
+
async computeHmac(value) {
|
|
1377
|
+
const subtle = globalThis.crypto?.subtle;
|
|
1378
|
+
if (!subtle) return null;
|
|
1379
|
+
this.apiHmacKey ??= subtle.importKey("raw", new TextEncoder().encode(this.options.apiKey), {
|
|
1380
|
+
name: "HMAC",
|
|
1381
|
+
hash: "SHA-256"
|
|
1382
|
+
}, false, ["sign"]).catch(() => null);
|
|
1383
|
+
const key = await this.apiHmacKey;
|
|
1384
|
+
if (!key) return null;
|
|
1385
|
+
const sig = new Uint8Array(await subtle.sign("HMAC", key, new TextEncoder().encode(value)));
|
|
1386
|
+
return Array.from(sig.slice(0, 16), (b) => b.toString(16).padStart(2, "0")).join("");
|
|
1387
|
+
}
|
|
1388
|
+
/** Decide what a request is (synchronously, so the visit cookie can still be set). */
|
|
1389
|
+
decide(method, url, header, ip) {
|
|
1390
|
+
const detection = detectFromHeaders(header, url);
|
|
1391
|
+
const ignore = {
|
|
1392
|
+
kind: "ignore",
|
|
1393
|
+
detection
|
|
1394
|
+
};
|
|
1395
|
+
if (!this.hitsOn && !this.apiOn) return ignore;
|
|
1396
|
+
const m = method.toUpperCase();
|
|
1397
|
+
if (m === "OPTIONS" || m === "CONNECT" || m === "TRACE") return ignore;
|
|
1398
|
+
let path;
|
|
1399
|
+
try {
|
|
1400
|
+
path = new URL(url).pathname;
|
|
1401
|
+
} catch {
|
|
1402
|
+
return ignore;
|
|
1403
|
+
}
|
|
1404
|
+
if (ASSET.test(path) || path.startsWith("/_next/") || path.startsWith("/_nuxt/") || path.startsWith("/_app/immutable/") || path.startsWith("/_astro/")) return ignore;
|
|
1405
|
+
const ig = this.options.ignore;
|
|
1406
|
+
if (ig && (typeof ig === "function" ? ig(path) : ig.test(path))) return ignore;
|
|
1407
|
+
if (this.apiOn && this.isApiPath(path)) return this.decideApi(m, url, path, header, ip, detection);
|
|
1408
|
+
if (!this.hitsOn) return ignore;
|
|
1409
|
+
const ua = header("user-agent") ?? null;
|
|
1410
|
+
if (!ip && !ua?.trim()) return ignore;
|
|
1411
|
+
const signed = detection.signed;
|
|
1412
|
+
if (signed || detection.isBot || ua && BOT_HINT.test(ua)) {
|
|
1413
|
+
const record = {
|
|
1414
|
+
ts: Date.now(),
|
|
1415
|
+
method: m,
|
|
1416
|
+
url,
|
|
1417
|
+
status: null,
|
|
1418
|
+
ua,
|
|
1419
|
+
ip,
|
|
1420
|
+
referrer: header("referer") ?? null
|
|
1421
|
+
};
|
|
1422
|
+
if (signed) record.headers = signatureHeaders(header);
|
|
1423
|
+
if (/text\/markdown/i.test(header("accept") ?? "") || /\.md$/i.test(path)) record.accept_md = true;
|
|
1424
|
+
const d = {
|
|
1425
|
+
kind: "bot",
|
|
1426
|
+
detection,
|
|
1427
|
+
record
|
|
1428
|
+
};
|
|
1429
|
+
if (this.options.visitCookie && m === "GET" && (signed || detection.browser)) {
|
|
1430
|
+
d.visitId = randomId();
|
|
1431
|
+
record.visit_id = d.visitId;
|
|
1432
|
+
d.cookie = `${VISIT_COOKIE}=${d.visitId}; Path=/; Max-Age=1800; SameSite=Lax${url.startsWith("https:") ? "; Secure" : ""}`;
|
|
1433
|
+
}
|
|
1434
|
+
return d;
|
|
1435
|
+
}
|
|
1436
|
+
if (m === "GET" && detection.referral) {
|
|
1437
|
+
let refHost = "";
|
|
1438
|
+
try {
|
|
1439
|
+
refHost = new URL(header("referer") ?? "").hostname.replace(/^www\./, "");
|
|
1440
|
+
} catch {}
|
|
1441
|
+
return {
|
|
1442
|
+
kind: "referral",
|
|
1443
|
+
detection,
|
|
1444
|
+
record: {
|
|
1445
|
+
ts: Date.now(),
|
|
1446
|
+
method: m,
|
|
1447
|
+
url,
|
|
1448
|
+
status: null,
|
|
1449
|
+
ua,
|
|
1450
|
+
ip: null,
|
|
1451
|
+
referrer: refHost ? `https://${refHost}/` : null
|
|
1452
|
+
}
|
|
1453
|
+
};
|
|
1454
|
+
}
|
|
1455
|
+
if (!this.options.countHumans || m !== "GET") return ignore;
|
|
1456
|
+
const dest = header("sec-fetch-dest");
|
|
1457
|
+
return (dest ? dest === "document" : (header("accept") ?? "").includes("text/html")) ? {
|
|
1458
|
+
kind: "human",
|
|
1459
|
+
detection
|
|
1460
|
+
} : ignore;
|
|
1461
|
+
}
|
|
1462
|
+
isApiPath(path) {
|
|
1463
|
+
const r = this.options.apiRoutes;
|
|
1464
|
+
if (r) return typeof r === "function" ? r(path) : r.test(path);
|
|
1465
|
+
return !this.hitsOn || API_PATH.test(path);
|
|
1466
|
+
}
|
|
1467
|
+
/** An API call: by an agent (kept, with how we know) or not (only counted). */
|
|
1468
|
+
decideApi(m, url, path, header, ip, detection) {
|
|
1469
|
+
const agent = header("x-ai-agent")?.trim().slice(0, 120) || null;
|
|
1470
|
+
const signed = detection.signed;
|
|
1471
|
+
let caller = null;
|
|
1472
|
+
const pick = this.options.apiCaller;
|
|
1473
|
+
if (pick !== false && (pick || signed || detection.isAI || agent)) try {
|
|
1474
|
+
const c = pick ? pick(header) : defaultCaller(header);
|
|
1475
|
+
caller = typeof c === "string" ? c ? { id: c } : null : c?.id ? c : null;
|
|
1476
|
+
} catch (err) {
|
|
1477
|
+
this.log("apiCaller failed", err);
|
|
1478
|
+
}
|
|
1479
|
+
if (!signed && !detection.isAI && !agent && !caller?.agent) return {
|
|
1480
|
+
kind: "api-other",
|
|
1481
|
+
detection
|
|
1482
|
+
};
|
|
1483
|
+
return {
|
|
1484
|
+
kind: "api",
|
|
1485
|
+
detection,
|
|
1486
|
+
api: {
|
|
1487
|
+
ts: Date.now(),
|
|
1488
|
+
method: m,
|
|
1489
|
+
route: routeFromPath(path),
|
|
1490
|
+
status: null,
|
|
1491
|
+
durationMs: null,
|
|
1492
|
+
ua: header("user-agent") ?? null,
|
|
1493
|
+
ip: signed || detection.isAI ? ip : null,
|
|
1494
|
+
url: signed ? url : null,
|
|
1495
|
+
headers: signed ? signatureHeaders(header) : {},
|
|
1496
|
+
agent,
|
|
1497
|
+
run: header("x-ai-agent-run")?.trim().slice(0, 200) || null,
|
|
1498
|
+
actor: null,
|
|
1499
|
+
actorAgent: caller?.agent?.slice(0, 120) ?? null
|
|
1500
|
+
},
|
|
1501
|
+
caller: caller?.id ?? null,
|
|
1502
|
+
started: now()
|
|
1503
|
+
};
|
|
1504
|
+
}
|
|
1505
|
+
/** Note whether the response to a bot was Markdown, from its content type (when the adapter can see it). */
|
|
1506
|
+
setContentType(d, contentType) {
|
|
1507
|
+
if (!d?.record || d.kind !== "bot" || typeof contentType !== "string" || !contentType) return;
|
|
1508
|
+
d.record.served_md = /text\/(?:x-)?markdown/i.test(contentType);
|
|
1509
|
+
}
|
|
1510
|
+
/** Use the framework's route pattern for an API call when it fits the request's path. */
|
|
1511
|
+
setRoute(d, pattern) {
|
|
1512
|
+
if (!d?.api) return;
|
|
1513
|
+
const fitted = fitRoute(pattern, d.api.route);
|
|
1514
|
+
if (fitted) d.api.route = fitted;
|
|
1515
|
+
}
|
|
1516
|
+
/** Queue a decided request once its status is known. */
|
|
1517
|
+
commit(d, status, waitUntil = ambientWaitUntil()) {
|
|
1518
|
+
if (d.kind === "ignore") return;
|
|
1519
|
+
if (d.kind === "human") {
|
|
1520
|
+
const k = day(Date.now());
|
|
1521
|
+
this.humans[k] = (this.humans[k] ?? 0) + 1;
|
|
1522
|
+
this.humanCount++;
|
|
1523
|
+
} else if (d.kind === "api-other") {
|
|
1524
|
+
const k = day(Date.now());
|
|
1525
|
+
this.apiOther[k] = (this.apiOther[k] ?? 0) + 1;
|
|
1526
|
+
this.apiOtherCount++;
|
|
1527
|
+
} else if (d.api) {
|
|
1528
|
+
d.api.status = status;
|
|
1529
|
+
d.api.durationMs = Math.round((now() - (d.started ?? now())) * 100) / 100;
|
|
1530
|
+
this.apiQueue.push({
|
|
1531
|
+
record: d.api,
|
|
1532
|
+
caller: d.caller ?? null
|
|
1533
|
+
});
|
|
1534
|
+
this.log("api", d.api.method, d.api.route, status);
|
|
1535
|
+
} else if (d.record) {
|
|
1536
|
+
d.record.status = status;
|
|
1537
|
+
this.queue.push(d.record);
|
|
1538
|
+
this.log(d.kind, d.record.ua, d.record.url);
|
|
1539
|
+
}
|
|
1540
|
+
if (waitUntil) {
|
|
1541
|
+
if (this.queue.length || this.apiQueue.length || this.humanCount + this.apiOtherCount >= 25 || Date.now() - this.lastFlush > 3e4) waitUntil(this.flush());
|
|
1542
|
+
return;
|
|
1543
|
+
}
|
|
1544
|
+
if (this.queue.length + this.apiQueue.length >= this.options.maxBatch) this.flush();
|
|
1545
|
+
else if (!this.timer) {
|
|
1546
|
+
this.timer = setTimeout(() => void this.flush(), this.options.flushIntervalMs);
|
|
1547
|
+
this.timer.unref?.();
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
fetchDecision(request, ip) {
|
|
1551
|
+
const h = request.headers;
|
|
1552
|
+
const get = (name) => h.get(name);
|
|
1553
|
+
try {
|
|
1554
|
+
return this.decide(request.method, request.url, get, ip ?? this.fetchIp(get));
|
|
1555
|
+
} catch (err) {
|
|
1556
|
+
this.log("track failed", err);
|
|
1557
|
+
return {
|
|
1558
|
+
kind: "ignore",
|
|
1559
|
+
detection: detectFromHeaders(get, request.url)
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
fetchFinish(d, status, response, waitUntil) {
|
|
1564
|
+
if (d.kind === "ignore") return;
|
|
1565
|
+
try {
|
|
1566
|
+
this.setContentType(d, response?.headers?.get?.("content-type"));
|
|
1567
|
+
if (d.cookie && response) try {
|
|
1568
|
+
response.headers.append("set-cookie", d.cookie);
|
|
1569
|
+
} catch {}
|
|
1570
|
+
this.commit(d, status, waitUntil ?? ambientWaitUntil());
|
|
1571
|
+
} catch (err) {
|
|
1572
|
+
this.log("track failed", err);
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
h3Decision(event) {
|
|
1576
|
+
const req = event.node?.req;
|
|
1577
|
+
const get = typeof event.headers?.get === "function" ? (name) => event.headers.get(name) : (name) => headerValue(req?.headers[name]);
|
|
1578
|
+
try {
|
|
1579
|
+
if (get("x-nitro-prerender") || (event.path ?? req?.url ?? "").startsWith("/__nuxt_error")) return {
|
|
1580
|
+
kind: "ignore",
|
|
1581
|
+
detection: detectFromHeaders(get, null)
|
|
1582
|
+
};
|
|
1583
|
+
const native = !!req?.socket?.remoteAddress;
|
|
1584
|
+
const url = native && req ? this.nodeUrl(req) : event.web?.request?.url ?? `${firstIp(get("x-forwarded-proto")) || "https"}://${get("host") ?? "localhost"}${event.path ?? req?.url ?? "/"}`;
|
|
1585
|
+
return this.decide(event.method ?? req?.method ?? "GET", url, get, native && req ? this.nodeIp(req) : this.fetchIp(get));
|
|
1586
|
+
} catch (err) {
|
|
1587
|
+
this.log("track failed", err);
|
|
1588
|
+
return {
|
|
1589
|
+
kind: "ignore",
|
|
1590
|
+
detection: detectFromHeaders(get, null)
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
/** Client IP of a fetch-API request: the configured header, else the platform's, else the usual forwarding headers. */
|
|
1595
|
+
fetchIp(h) {
|
|
1596
|
+
if (this.options.ipHeader) return firstIp(h(this.options.ipHeader.toLowerCase()));
|
|
1597
|
+
const platform = platformIpHeader();
|
|
1598
|
+
return platform && firstIp(h(platform)) || forwardedIp(h);
|
|
1599
|
+
}
|
|
1600
|
+
/** Whether this request came through a proxy whose forwarding headers we should believe. */
|
|
1601
|
+
viaProxy(req) {
|
|
1602
|
+
return this.options.trustProxy ?? isPrivateIp(req.socket?.remoteAddress);
|
|
1603
|
+
}
|
|
1604
|
+
nodeUrl(req) {
|
|
1605
|
+
const h = (n) => headerValue(req.headers[n]);
|
|
1606
|
+
const proxied = this.viaProxy(req);
|
|
1607
|
+
return `${proxied && firstIp(h("x-forwarded-proto")) || (req.socket?.encrypted ? "https" : "http")}://${proxied && firstIp(h("x-forwarded-host")) || h("host") || "localhost"}${req.originalUrl ?? req.url ?? "/"}`;
|
|
1608
|
+
}
|
|
1609
|
+
nodeIp(req) {
|
|
1610
|
+
const h = (n) => headerValue(req.headers[n]);
|
|
1611
|
+
if (this.options.ipHeader) return firstIp(h(this.options.ipHeader.toLowerCase())) ?? req.socket?.remoteAddress ?? null;
|
|
1612
|
+
if (this.viaProxy(req)) {
|
|
1613
|
+
const platform = platformIpHeader();
|
|
1614
|
+
const fwd = platform && firstIp(h(platform)) || firstIp(h("cf-connecting-ip") ?? h("true-client-ip") ?? h("x-real-ip") ?? h("fly-client-ip") ?? h("x-forwarded-for"));
|
|
1615
|
+
if (fwd) return fwd;
|
|
1616
|
+
}
|
|
1617
|
+
return req.socket?.remoteAddress ?? null;
|
|
1618
|
+
}
|
|
1619
|
+
log(...args) {
|
|
1620
|
+
if (this.options.debug) console.debug("[agentstats]", ...args);
|
|
1621
|
+
}
|
|
1622
|
+
};
|
|
1623
|
+
const DECISION = Symbol.for("agentstats.decision");
|
|
1624
|
+
function headerValue(v) {
|
|
1625
|
+
if (v === void 0) return null;
|
|
1626
|
+
return Array.isArray(v) ? v.join(", ") : v;
|
|
1627
|
+
}
|
|
1628
|
+
function appendNodeCookie(res, cookie) {
|
|
1629
|
+
if (res.headersSent || !res.setHeader) return;
|
|
1630
|
+
const prev = res.getHeader?.("set-cookie");
|
|
1631
|
+
const list = prev === void 0 ? [] : Array.isArray(prev) ? prev.map(String) : [String(prev)];
|
|
1632
|
+
res.setHeader("set-cookie", [...list, cookie]);
|
|
1633
|
+
}
|
|
1634
|
+
function createAgentStats(options) {
|
|
1635
|
+
return new AgentStatsServer(options);
|
|
1636
|
+
}
|
|
1637
|
+
//#endregion
|
|
1638
|
+
export { AI_CATEGORIES, AgentStatsServer, BOT_PATTERNS, BOT_PRODUCTS, SERVER_SDK_VERSION, classifyReferral, classifyUserAgent, createAgentStats, createAgentStats as default, detectBot, isAiCategory, isPrivateIp, normalizeRoutePattern, routeFromPath };
|