@waniwani/relay 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -6
- package/dist/index.d.ts +21 -4
- package/dist/index.js +581 -163
- package/dist/index.js.map +1 -1
- package/package.json +3 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,213 @@
|
|
|
1
|
+
// src/settings.ts
|
|
2
|
+
var settings = {
|
|
3
|
+
apiUrls: {
|
|
4
|
+
us: "https://app.waniwani.ai",
|
|
5
|
+
eu: "https://eu.app.waniwani.ai"
|
|
6
|
+
},
|
|
7
|
+
requestTimeoutMs: 2e3,
|
|
8
|
+
cloudFrontTimeoutMs: 1e3,
|
|
9
|
+
retryDelayMs: 500,
|
|
10
|
+
rulesRefreshMs: 12 * 60 * 60 * 1e3,
|
|
11
|
+
rulesRetryMs: 5 * 60 * 1e3,
|
|
12
|
+
siteRulesTimeoutMs: 300,
|
|
13
|
+
siteRulesRefreshMs: 6e4,
|
|
14
|
+
siteRulesRetryMs: 5e3
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
// src/version.ts
|
|
18
|
+
var RELAY_VERSION = true ? "0.4.0" : "0.0.0-dev";
|
|
19
|
+
|
|
20
|
+
// src/agent-page.ts
|
|
21
|
+
var isObject = (value) => typeof value === "object" && value !== null;
|
|
22
|
+
var isStringArray = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
23
|
+
var isPositive = (value) => typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
24
|
+
function parseSiteRules(body) {
|
|
25
|
+
if (!isObject(body) || body.version !== 1) {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const { pagesUrl, userAgentTokens, deadlines, maxBodyBytes, sites } = body;
|
|
29
|
+
if (typeof pagesUrl !== "string" || !isStringArray(userAgentTokens) || !isObject(deadlines) || !isPositive(deadlines.headersMs) || !isPositive(deadlines.bodyMs) || !isPositive(maxBodyBytes) || !Array.isArray(sites)) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
const parsedSites = [];
|
|
33
|
+
for (const site of sites) {
|
|
34
|
+
if (!isObject(site) || typeof site.hostname !== "string" || !isObject(site.agentPages) || typeof site.agentPages.enabled !== "boolean" || !isStringArray(site.agentPages.paths)) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
parsedSites.push({
|
|
38
|
+
hostname: site.hostname.toLowerCase(),
|
|
39
|
+
agentPages: {
|
|
40
|
+
enabled: site.agentPages.enabled,
|
|
41
|
+
paths: site.agentPages.paths
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
version: 1,
|
|
47
|
+
pagesUrl: pagesUrl.replace(/\/+$/, ""),
|
|
48
|
+
userAgentTokens: userAgentTokens.filter((token) => token !== ""),
|
|
49
|
+
deadlines: { headersMs: deadlines.headersMs, bodyMs: deadlines.bodyMs },
|
|
50
|
+
maxBodyBytes,
|
|
51
|
+
sites: parsedSites
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
var cache = /* @__PURE__ */ new Map();
|
|
55
|
+
async function fetchSiteRules(url, timeoutMs) {
|
|
56
|
+
const response = await fetch(url, {
|
|
57
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
58
|
+
});
|
|
59
|
+
if (!response.ok) {
|
|
60
|
+
await response.body?.cancel().catch(() => {
|
|
61
|
+
});
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const body = await response.json();
|
|
65
|
+
return parseSiteRules(isObject(body) ? body.data : null);
|
|
66
|
+
}
|
|
67
|
+
function startFetch(entry, url, timeoutMs) {
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
entry.lastAttemptAt = now;
|
|
70
|
+
entry.lastAttemptOk = false;
|
|
71
|
+
const work = fetchSiteRules(url, timeoutMs).catch(() => null).then((rules) => {
|
|
72
|
+
if (rules) {
|
|
73
|
+
entry.rules = rules;
|
|
74
|
+
entry.fetchedAt = Date.now();
|
|
75
|
+
entry.lastAttemptOk = true;
|
|
76
|
+
}
|
|
77
|
+
entry.inFlight = null;
|
|
78
|
+
return rules;
|
|
79
|
+
});
|
|
80
|
+
entry.inFlight = work;
|
|
81
|
+
return work;
|
|
82
|
+
}
|
|
83
|
+
async function loadSiteRules(apiUrl, ingestToken, background) {
|
|
84
|
+
const url = `${apiUrl}/api/v2/agent-traffic/serve/${ingestToken}/rules`;
|
|
85
|
+
let entry = cache.get(url);
|
|
86
|
+
if (!entry) {
|
|
87
|
+
entry = {
|
|
88
|
+
rules: null,
|
|
89
|
+
fetchedAt: 0,
|
|
90
|
+
lastAttemptAt: Number.NEGATIVE_INFINITY,
|
|
91
|
+
lastAttemptOk: false,
|
|
92
|
+
inFlight: null
|
|
93
|
+
};
|
|
94
|
+
cache.set(url, entry);
|
|
95
|
+
}
|
|
96
|
+
const now = Date.now();
|
|
97
|
+
if (entry.rules) {
|
|
98
|
+
const stale = now - entry.fetchedAt >= settings.siteRulesRefreshMs;
|
|
99
|
+
const retryDue = entry.lastAttemptOk || now - entry.lastAttemptAt >= settings.siteRulesRetryMs;
|
|
100
|
+
if (stale && retryDue && !entry.inFlight) {
|
|
101
|
+
background(startFetch(entry, url, settings.requestTimeoutMs));
|
|
102
|
+
}
|
|
103
|
+
return entry.rules;
|
|
104
|
+
}
|
|
105
|
+
if (entry.inFlight) {
|
|
106
|
+
return entry.inFlight;
|
|
107
|
+
}
|
|
108
|
+
if (now - entry.lastAttemptAt < settings.siteRulesRetryMs) {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
return startFetch(entry, url, settings.siteRulesTimeoutMs);
|
|
112
|
+
}
|
|
113
|
+
function normalizePath(pathname) {
|
|
114
|
+
const trimmed = pathname.replace(/\/+$/, "");
|
|
115
|
+
return trimmed === "" ? "/" : trimmed;
|
|
116
|
+
}
|
|
117
|
+
function matchesPath(path, patterns) {
|
|
118
|
+
return patterns.some((pattern) => {
|
|
119
|
+
if (pattern === "*" || pattern === "/*") {
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
if (pattern.endsWith("/*")) {
|
|
123
|
+
const folder = normalizePath(pattern.slice(0, -2));
|
|
124
|
+
return path === folder || path.startsWith(`${folder}/`);
|
|
125
|
+
}
|
|
126
|
+
return path === normalizePath(pattern);
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
var TIMEOUT = /* @__PURE__ */ Symbol("timeout");
|
|
130
|
+
function within(promise, ms) {
|
|
131
|
+
let timer;
|
|
132
|
+
const deadline = new Promise((resolve) => {
|
|
133
|
+
timer = setTimeout(() => resolve(TIMEOUT), ms);
|
|
134
|
+
});
|
|
135
|
+
return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
|
|
136
|
+
}
|
|
137
|
+
async function readCapped(body, deadline, maxBytes) {
|
|
138
|
+
const reader = body.getReader();
|
|
139
|
+
const chunks = [];
|
|
140
|
+
let size = 0;
|
|
141
|
+
try {
|
|
142
|
+
while (true) {
|
|
143
|
+
const result = await Promise.race([reader.read(), deadline]);
|
|
144
|
+
if (result === TIMEOUT) {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
if (result.done) {
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
size += result.value.byteLength;
|
|
151
|
+
if (size > maxBytes) {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
chunks.push(result.value);
|
|
155
|
+
}
|
|
156
|
+
} finally {
|
|
157
|
+
reader.cancel().catch(() => {
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
const out = new Uint8Array(size);
|
|
161
|
+
let offset = 0;
|
|
162
|
+
for (const chunk of chunks) {
|
|
163
|
+
out.set(chunk, offset);
|
|
164
|
+
offset += chunk.byteLength;
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
async function fetchAgentPage(rules, hostname, path) {
|
|
169
|
+
const controller = new AbortController();
|
|
170
|
+
try {
|
|
171
|
+
const response = await within(
|
|
172
|
+
fetch(`${rules.pagesUrl}/${hostname}${path}`, {
|
|
173
|
+
headers: {
|
|
174
|
+
accept: "text/markdown, text/html;q=0.9",
|
|
175
|
+
"x-waniwani-relay": RELAY_VERSION
|
|
176
|
+
},
|
|
177
|
+
signal: controller.signal
|
|
178
|
+
}),
|
|
179
|
+
rules.deadlines.headersMs
|
|
180
|
+
);
|
|
181
|
+
if (response === TIMEOUT) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
185
|
+
if (response.status !== 200 || !contentType.toLowerCase().startsWith("text/") || !response.body) {
|
|
186
|
+
await response.body?.cancel().catch(() => {
|
|
187
|
+
});
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
let timer;
|
|
191
|
+
const deadline = new Promise((resolve) => {
|
|
192
|
+
timer = setTimeout(() => resolve(TIMEOUT), rules.deadlines.bodyMs);
|
|
193
|
+
});
|
|
194
|
+
try {
|
|
195
|
+
const body = await readCapped(
|
|
196
|
+
response.body,
|
|
197
|
+
deadline,
|
|
198
|
+
rules.maxBodyBytes
|
|
199
|
+
);
|
|
200
|
+
return body ? { body, contentType } : null;
|
|
201
|
+
} finally {
|
|
202
|
+
clearTimeout(timer);
|
|
203
|
+
}
|
|
204
|
+
} catch {
|
|
205
|
+
return null;
|
|
206
|
+
} finally {
|
|
207
|
+
controller.abort();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
1
211
|
// src/headers.ts
|
|
2
212
|
var HEADER_CAP = [
|
|
3
213
|
"user-agent",
|
|
@@ -64,17 +274,6 @@ function clientIp(headers) {
|
|
|
64
274
|
return ip ? ip : null;
|
|
65
275
|
}
|
|
66
276
|
|
|
67
|
-
// src/settings.ts
|
|
68
|
-
var settings = {
|
|
69
|
-
apiUrls: {
|
|
70
|
-
us: "https://app.waniwani.ai",
|
|
71
|
-
eu: "https://eu.app.waniwani.ai"
|
|
72
|
-
},
|
|
73
|
-
requestTimeoutMs: 2e3,
|
|
74
|
-
cloudFrontTimeoutMs: 1e3,
|
|
75
|
-
retryDelayMs: 500
|
|
76
|
-
};
|
|
77
|
-
|
|
78
277
|
// src/http.ts
|
|
79
278
|
async function fetchWithTimeout(url, init, timeoutMs = settings.requestTimeoutMs) {
|
|
80
279
|
const controller = new AbortController();
|
|
@@ -151,165 +350,179 @@ function readEnv(name) {
|
|
|
151
350
|
// src/logic/decide.ts
|
|
152
351
|
import { BlockList } from "node:net";
|
|
153
352
|
|
|
154
|
-
// src/logic/agents.json
|
|
155
|
-
var
|
|
156
|
-
source: "
|
|
353
|
+
// src/logic/agents.generated.json
|
|
354
|
+
var agents_generated_default = {
|
|
355
|
+
source: "Generated by scripts/build-rules.ts from the WaniWani rules file d9b242ef451056c0, do not edit. Each token is matched case-insensitively inside the User-Agent header.",
|
|
157
356
|
tokens: [
|
|
158
|
-
"
|
|
159
|
-
"
|
|
160
|
-
"
|
|
161
|
-
"googleother",
|
|
162
|
-
"googleother-image",
|
|
163
|
-
"googleother-video",
|
|
164
|
-
"meta-externalagent",
|
|
165
|
-
"facebookbot",
|
|
166
|
-
"bytespider",
|
|
167
|
-
"tiktokspider",
|
|
168
|
-
"ccbot",
|
|
169
|
-
"diffbot",
|
|
357
|
+
"addsearchbot",
|
|
358
|
+
"agentdatabot",
|
|
359
|
+
"agenttimes",
|
|
170
360
|
"ai2bot",
|
|
361
|
+
"ai2bot-deepresearcheval",
|
|
171
362
|
"ai2bot-dolma",
|
|
172
|
-
"timpibot",
|
|
173
|
-
"cohere-training-data-crawler",
|
|
174
|
-
"mistralai-training",
|
|
175
|
-
"deepseekbot",
|
|
176
|
-
"erniebot",
|
|
177
|
-
"qwenbot",
|
|
178
|
-
"pangubot",
|
|
179
|
-
"kimibot",
|
|
180
|
-
"yandexadditional",
|
|
181
|
-
"yandexadditionalbot",
|
|
182
|
-
"brightbot",
|
|
183
|
-
"reflectionbot",
|
|
184
|
-
"agenttimes",
|
|
185
363
|
"aihitbot",
|
|
364
|
+
"aiwebindex",
|
|
365
|
+
"allenai.org",
|
|
366
|
+
"amazon-kendra",
|
|
367
|
+
"amazon-qbusiness",
|
|
368
|
+
"amazonbot",
|
|
369
|
+
"amazonbuyforme",
|
|
370
|
+
"amzn-searchbot",
|
|
371
|
+
"amzn-user",
|
|
372
|
+
"andibot",
|
|
373
|
+
"anthropic-ai",
|
|
374
|
+
"apifybot",
|
|
375
|
+
"apifywebsitecontentcrawler",
|
|
376
|
+
"applebot",
|
|
377
|
+
"aranet-searchbot",
|
|
378
|
+
"atlassian-bot",
|
|
186
379
|
"awario",
|
|
380
|
+
"azureai-searchbot",
|
|
381
|
+
"bedrockbot",
|
|
382
|
+
"bigsur.ai",
|
|
383
|
+
"bixelbot",
|
|
384
|
+
"bravebot",
|
|
385
|
+
"brightbot",
|
|
386
|
+
"buddybot",
|
|
387
|
+
"bytespider",
|
|
388
|
+
"ccbot",
|
|
389
|
+
"channel3bot",
|
|
187
390
|
"chatglm-spider",
|
|
391
|
+
"chatgpt agent",
|
|
392
|
+
"chatgpt-user",
|
|
393
|
+
"claude-code",
|
|
394
|
+
"claude-searchbot",
|
|
395
|
+
"claude-user",
|
|
396
|
+
"claude-web",
|
|
397
|
+
"claudebot",
|
|
398
|
+
"cloudflare-autorag",
|
|
399
|
+
"cloudflarebrowserrenderingcrawler",
|
|
400
|
+
"cloudvertexbot",
|
|
401
|
+
"cohere-ai",
|
|
402
|
+
"cohere-training-data-crawler",
|
|
188
403
|
"cotoyogi",
|
|
404
|
+
"coveobot",
|
|
405
|
+
"cragcrawler",
|
|
406
|
+
"crawl4ai",
|
|
407
|
+
"crawlspace",
|
|
408
|
+
"cursor/1.0.0",
|
|
189
409
|
"datenbank crawler",
|
|
410
|
+
"deepseekbot",
|
|
411
|
+
"diffbot",
|
|
412
|
+
"diffbot-user",
|
|
413
|
+
"doubaobot",
|
|
414
|
+
"duckassistbot",
|
|
190
415
|
"echobot bot",
|
|
191
416
|
"echoboxbot",
|
|
417
|
+
"erniebot",
|
|
418
|
+
"exabot",
|
|
419
|
+
"exasearchbot",
|
|
420
|
+
"exauser",
|
|
421
|
+
"facebookbot",
|
|
192
422
|
"factset_spyderbot",
|
|
423
|
+
"firecrawlagent",
|
|
193
424
|
"friendlycrawler",
|
|
425
|
+
"geisthaus-pagefetcher",
|
|
426
|
+
"gemini-deep-research",
|
|
427
|
+
"google-ads-creatives-assistant",
|
|
428
|
+
"google-agent",
|
|
429
|
+
"google-cloudvertexbot",
|
|
430
|
+
"google-firebase",
|
|
431
|
+
"google-gemini-cli",
|
|
432
|
+
"google-gemininotebook",
|
|
433
|
+
"google-lens",
|
|
434
|
+
"google-notebooklm",
|
|
435
|
+
"googleagent-mariner",
|
|
436
|
+
"googleagent-urlcontext",
|
|
437
|
+
"googleother",
|
|
438
|
+
"googleother-image",
|
|
439
|
+
"googleother-video",
|
|
440
|
+
"gptbot",
|
|
441
|
+
"grove/1.0 rss feed service",
|
|
442
|
+
"henkbot",
|
|
443
|
+
"iaskbot",
|
|
444
|
+
"iaskspider",
|
|
194
445
|
"icc-crawler",
|
|
195
446
|
"imagesiftbot",
|
|
196
447
|
"imagespider",
|
|
197
448
|
"img2dataset",
|
|
198
449
|
"isscyberriskcrawler",
|
|
450
|
+
"jackjillcareersbot",
|
|
451
|
+
"kagi-fetcher",
|
|
199
452
|
"kangaroo bot",
|
|
453
|
+
"keenablebot",
|
|
454
|
+
"kernel searchbot",
|
|
455
|
+
"kimi-agent",
|
|
456
|
+
"kimi-searchbot",
|
|
457
|
+
"kimi-user",
|
|
458
|
+
"kimibot",
|
|
459
|
+
"klaviyoaibot",
|
|
200
460
|
"kunatocrawler",
|
|
201
461
|
"laion-huggingface-processor",
|
|
202
462
|
"laiondownloader",
|
|
463
|
+
"lightpanda",
|
|
464
|
+
"linerbot",
|
|
465
|
+
"linkupbot",
|
|
466
|
+
"manus-user",
|
|
467
|
+
"meta-externalagent",
|
|
468
|
+
"meta-externalfetcher",
|
|
469
|
+
"meta-webindexer",
|
|
470
|
+
"mistralai-index",
|
|
471
|
+
"mistralai-training",
|
|
472
|
+
"mistralai-user",
|
|
473
|
+
"momentic",
|
|
474
|
+
"mozilla-tabstack",
|
|
203
475
|
"mycentralaiscraperbot",
|
|
204
476
|
"nagetbot",
|
|
205
477
|
"netestate imprint crawler",
|
|
478
|
+
"notebooklm",
|
|
479
|
+
"novaact",
|
|
480
|
+
"oai-adsbot",
|
|
481
|
+
"oai-searchbot",
|
|
206
482
|
"omgili",
|
|
207
483
|
"omgilibot",
|
|
484
|
+
"pangubot",
|
|
208
485
|
"panscient",
|
|
209
|
-
"
|
|
210
|
-
"sbintuitionsbot",
|
|
211
|
-
"sidetrade indexer bot",
|
|
212
|
-
"thinkbot",
|
|
213
|
-
"velenpublicwebcrawler",
|
|
214
|
-
"wardbot",
|
|
215
|
-
"oai-searchbot",
|
|
216
|
-
"oai-adsbot",
|
|
217
|
-
"claude-searchbot",
|
|
486
|
+
"perplexity-user",
|
|
218
487
|
"perplexitybot",
|
|
219
|
-
"applebot",
|
|
220
|
-
"amazonbot",
|
|
221
|
-
"amzn-searchbot",
|
|
222
|
-
"amazon-kendra",
|
|
223
|
-
"meta-webindexer",
|
|
224
|
-
"mistralai-index",
|
|
225
|
-
"kimi-searchbot",
|
|
226
|
-
"azureai-searchbot",
|
|
227
488
|
"petalbot",
|
|
228
|
-
"
|
|
229
|
-
"
|
|
230
|
-
"
|
|
231
|
-
"
|
|
232
|
-
"
|
|
489
|
+
"phindbot",
|
|
490
|
+
"poggio-citations",
|
|
491
|
+
"poseidon research crawler",
|
|
492
|
+
"qodercli",
|
|
493
|
+
"qualifiedbot",
|
|
494
|
+
"querit-searchbot",
|
|
495
|
+
"queritbot",
|
|
496
|
+
"quillbot",
|
|
497
|
+
"qwenbot",
|
|
498
|
+
"reflectionbot",
|
|
499
|
+
"ryebot",
|
|
500
|
+
"sbintuitionsbot",
|
|
233
501
|
"seltzsitemapfetcher",
|
|
502
|
+
"semanticscholarbot",
|
|
503
|
+
"semrushbot-ocob",
|
|
504
|
+
"semrushbot-swa",
|
|
505
|
+
"shap-user",
|
|
234
506
|
"shapbot",
|
|
507
|
+
"sidetrade indexer bot",
|
|
235
508
|
"tavilybot",
|
|
236
|
-
"bravebot",
|
|
237
|
-
"cloudflare-autorag",
|
|
238
|
-
"jackjillcareersbot",
|
|
239
|
-
"addsearchbot",
|
|
240
|
-
"aiwebindex",
|
|
241
|
-
"andibot",
|
|
242
|
-
"aranet-searchbot",
|
|
243
|
-
"channel3bot",
|
|
244
|
-
"iaskbot",
|
|
245
|
-
"iaskspider",
|
|
246
|
-
"querit-searchbot",
|
|
247
|
-
"queritbot",
|
|
248
|
-
"terracotta",
|
|
249
509
|
"terra cotta",
|
|
250
|
-
"
|
|
251
|
-
"
|
|
252
|
-
"
|
|
253
|
-
"
|
|
254
|
-
"claude-web",
|
|
255
|
-
"claude-code",
|
|
256
|
-
"perplexity-user",
|
|
257
|
-
"mistralai-user",
|
|
258
|
-
"meta-externalfetcher",
|
|
259
|
-
"google-cloudvertexbot",
|
|
260
|
-
"cloudvertexbot",
|
|
261
|
-
"google-firebase",
|
|
262
|
-
"google-notebooklm",
|
|
263
|
-
"notebooklm",
|
|
264
|
-
"gemini-deep-research",
|
|
265
|
-
"google-agent",
|
|
266
|
-
"googleagent-mariner",
|
|
267
|
-
"googleagent-urlcontext",
|
|
268
|
-
"google-gemini-cli",
|
|
269
|
-
"duckassistbot",
|
|
270
|
-
"cohere-ai",
|
|
271
|
-
"amzn-user",
|
|
272
|
-
"amazonbuyforme",
|
|
273
|
-
"amazon-qbusiness",
|
|
274
|
-
"novaact",
|
|
275
|
-
"bedrockbot",
|
|
276
|
-
"diffbot-user",
|
|
277
|
-
"ai2bot-deepresearcheval",
|
|
278
|
-
"kimi-agent",
|
|
279
|
-
"kimi-user",
|
|
280
|
-
"manus-user",
|
|
281
|
-
"kagi-fetcher",
|
|
282
|
-
"phindbot",
|
|
510
|
+
"terracotta",
|
|
511
|
+
"thinkbot",
|
|
512
|
+
"tiktokspider",
|
|
513
|
+
"timpibot",
|
|
283
514
|
"tongyibot",
|
|
284
|
-
"
|
|
285
|
-
"doubaobot",
|
|
286
|
-
"shap-user",
|
|
287
|
-
"firecrawlagent",
|
|
288
|
-
"apifybot",
|
|
289
|
-
"apifywebsitecontentcrawler",
|
|
290
|
-
"cloudflarebrowserrenderingcrawler",
|
|
291
|
-
"crawlspace",
|
|
292
|
-
"mozilla-tabstack",
|
|
293
|
-
"klaviyoaibot",
|
|
294
|
-
"atlassian-bot",
|
|
295
|
-
"semrushbot-ocob",
|
|
296
|
-
"semrushbot-swa",
|
|
297
|
-
"quillbot",
|
|
298
|
-
"bigsur.ai",
|
|
299
|
-
"geisthaus-pagefetcher",
|
|
300
|
-
"poggio-citations",
|
|
301
|
-
"qualifiedbot",
|
|
515
|
+
"twilioknowledge",
|
|
302
516
|
"twinagent",
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
"
|
|
306
|
-
"
|
|
307
|
-
"
|
|
308
|
-
"
|
|
309
|
-
"
|
|
310
|
-
"
|
|
311
|
-
"
|
|
312
|
-
"wrtnbot"
|
|
517
|
+
"v0bot",
|
|
518
|
+
"velenpublicwebcrawler",
|
|
519
|
+
"wardbot",
|
|
520
|
+
"wrtnbot",
|
|
521
|
+
"yandexadditional",
|
|
522
|
+
"yandexadditionalbot",
|
|
523
|
+
"yiyanbot",
|
|
524
|
+
"youbot",
|
|
525
|
+
"zanistabot"
|
|
313
526
|
]
|
|
314
527
|
};
|
|
315
528
|
|
|
@@ -629,6 +842,27 @@ var ip_ranges_generated_default = {
|
|
|
629
842
|
],
|
|
630
843
|
ipv6: []
|
|
631
844
|
},
|
|
845
|
+
{
|
|
846
|
+
bot: "ClaudeBot",
|
|
847
|
+
url: "https://claude.com/crawling/bots.json",
|
|
848
|
+
ipv4: [
|
|
849
|
+
"34.152.124.140/32",
|
|
850
|
+
"34.153.21.190/32",
|
|
851
|
+
"34.162.27.34/32",
|
|
852
|
+
"34.162.86.102/32",
|
|
853
|
+
"34.162.90.102/32",
|
|
854
|
+
"34.162.111.78/32",
|
|
855
|
+
"34.162.153.71/32",
|
|
856
|
+
"34.162.191.81/32",
|
|
857
|
+
"34.162.230.222/32",
|
|
858
|
+
"34.162.244.71/32",
|
|
859
|
+
"34.186.252.4/32",
|
|
860
|
+
"136.79.143.251/32",
|
|
861
|
+
"136.83.20.215/32",
|
|
862
|
+
"216.73.216.0/22"
|
|
863
|
+
],
|
|
864
|
+
ipv6: []
|
|
865
|
+
},
|
|
632
866
|
{
|
|
633
867
|
bot: "PerplexityBot",
|
|
634
868
|
url: "https://www.perplexity.com/perplexitybot.json",
|
|
@@ -1182,6 +1416,46 @@ var ip_ranges_generated_default = {
|
|
|
1182
1416
|
"191.235.202.48/32"
|
|
1183
1417
|
],
|
|
1184
1418
|
ipv6: []
|
|
1419
|
+
},
|
|
1420
|
+
{
|
|
1421
|
+
bot: "Applebot",
|
|
1422
|
+
url: "https://search.developer.apple.com/applebot.json",
|
|
1423
|
+
ipv4: [
|
|
1424
|
+
"17.166.20.0/24",
|
|
1425
|
+
"17.166.21.0/24",
|
|
1426
|
+
"17.166.22.0/24",
|
|
1427
|
+
"17.166.23.0/24",
|
|
1428
|
+
"17.166.24.0/24",
|
|
1429
|
+
"17.166.25.0/24",
|
|
1430
|
+
"17.166.150.0/24",
|
|
1431
|
+
"17.166.151.0/24",
|
|
1432
|
+
"17.166.152.0/24",
|
|
1433
|
+
"17.166.153.0/24",
|
|
1434
|
+
"17.166.154.0/24",
|
|
1435
|
+
"17.166.155.0/24",
|
|
1436
|
+
"17.166.192.48/28",
|
|
1437
|
+
"17.166.232.0/24",
|
|
1438
|
+
"17.166.233.0/24",
|
|
1439
|
+
"17.166.234.0/24",
|
|
1440
|
+
"17.166.235.0/24",
|
|
1441
|
+
"17.166.236.0/24",
|
|
1442
|
+
"17.166.237.0/24",
|
|
1443
|
+
"17.166.240.48/28",
|
|
1444
|
+
"17.166.248.48/28",
|
|
1445
|
+
"17.241.193.160/27",
|
|
1446
|
+
"17.241.200.160/27",
|
|
1447
|
+
"17.241.208.160/27"
|
|
1448
|
+
],
|
|
1449
|
+
ipv6: []
|
|
1450
|
+
},
|
|
1451
|
+
{
|
|
1452
|
+
bot: "Claude API",
|
|
1453
|
+
url: "https://platform.claude.com/docs/en/api/ip-addresses.md",
|
|
1454
|
+
ipv4: [
|
|
1455
|
+
"153.61.208.0/20",
|
|
1456
|
+
"160.79.104.0/21"
|
|
1457
|
+
],
|
|
1458
|
+
ipv6: []
|
|
1185
1459
|
}
|
|
1186
1460
|
]
|
|
1187
1461
|
};
|
|
@@ -1189,29 +1463,42 @@ var ip_ranges_generated_default = {
|
|
|
1189
1463
|
// src/logic/decide.ts
|
|
1190
1464
|
var MAX_SCANNED_USER_AGENT = 1024;
|
|
1191
1465
|
var PREFIX = 3;
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
const
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
list
|
|
1198
|
-
|
|
1199
|
-
|
|
1466
|
+
function compile(rules) {
|
|
1467
|
+
const tokensByPrefix = /* @__PURE__ */ new Map();
|
|
1468
|
+
for (const token of rules.tokens) {
|
|
1469
|
+
const key = token.slice(0, PREFIX);
|
|
1470
|
+
const list = tokensByPrefix.get(key);
|
|
1471
|
+
if (list) {
|
|
1472
|
+
list.push(token);
|
|
1473
|
+
} else {
|
|
1474
|
+
tokensByPrefix.set(key, [token]);
|
|
1475
|
+
}
|
|
1200
1476
|
}
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1477
|
+
const agentRanges = new BlockList();
|
|
1478
|
+
const addRanges = (cidrs, type) => {
|
|
1479
|
+
for (const cidr of cidrs) {
|
|
1480
|
+
const [address = "", prefix, extra] = cidr.split("/");
|
|
1481
|
+
if (!prefix || extra !== void 0) {
|
|
1482
|
+
throw new Error(`malformed range ${cidr}`);
|
|
1483
|
+
}
|
|
1484
|
+
agentRanges.addSubnet(address, Number(prefix), type);
|
|
1485
|
+
}
|
|
1486
|
+
};
|
|
1487
|
+
for (const source of rules.ipRanges) {
|
|
1488
|
+
addRanges(source.ipv4, "ipv4");
|
|
1489
|
+
addRanges(source.ipv6, "ipv6");
|
|
1207
1490
|
}
|
|
1208
|
-
};
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1491
|
+
return { tokensByPrefix, agentRanges };
|
|
1492
|
+
}
|
|
1493
|
+
var bundled = compile({ tokens: agents_generated_default.tokens, ipRanges: ip_ranges_generated_default.sources });
|
|
1494
|
+
var remote = null;
|
|
1495
|
+
function applyRules(rules) {
|
|
1496
|
+
remote = compile(rules);
|
|
1212
1497
|
}
|
|
1213
|
-
var
|
|
1214
|
-
|
|
1498
|
+
var pick = (useBundled) => useBundled ? bundled : remote ?? bundled;
|
|
1499
|
+
var isAgentIp = (ip, useBundled = false) => pick(useBundled).agentRanges.check(ip, ip.includes(":") ? "ipv6" : "ipv4");
|
|
1500
|
+
function hasAgentToken(userAgent, useBundled = false) {
|
|
1501
|
+
const { tokensByPrefix } = pick(useBundled);
|
|
1215
1502
|
const text = userAgent.slice(0, MAX_SCANNED_USER_AGENT).toLowerCase();
|
|
1216
1503
|
for (let i = 0; i + PREFIX <= text.length; i++) {
|
|
1217
1504
|
const candidates = tokensByPrefix.get(text.slice(i, i + PREFIX));
|
|
@@ -1225,9 +1512,13 @@ function hasAgentToken(userAgent) {
|
|
|
1225
1512
|
}
|
|
1226
1513
|
return false;
|
|
1227
1514
|
}
|
|
1228
|
-
function decide({
|
|
1515
|
+
function decide({
|
|
1516
|
+
headers,
|
|
1517
|
+
ip,
|
|
1518
|
+
useBundled = false
|
|
1519
|
+
}) {
|
|
1229
1520
|
const userAgent = headers["user-agent"];
|
|
1230
|
-
return userAgent !== void 0 && hasAgentToken(userAgent) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip);
|
|
1521
|
+
return userAgent !== void 0 && hasAgentToken(userAgent, useBundled) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip, useBundled);
|
|
1231
1522
|
}
|
|
1232
1523
|
|
|
1233
1524
|
// src/match-key.ts
|
|
@@ -1261,6 +1552,62 @@ async function deriveMatchKey(orgKey, ip) {
|
|
|
1261
1552
|
return toHex(signature);
|
|
1262
1553
|
}
|
|
1263
1554
|
|
|
1555
|
+
// src/refresh.ts
|
|
1556
|
+
var isObject2 = (value) => typeof value === "object" && value !== null;
|
|
1557
|
+
var isStringArray2 = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
1558
|
+
function parseRemoteRules(body) {
|
|
1559
|
+
if (!isObject2(body) || body.version !== 1) {
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1562
|
+
const { tokens, ipRanges } = body;
|
|
1563
|
+
if (!isStringArray2(tokens) || !Array.isArray(ipRanges)) {
|
|
1564
|
+
return null;
|
|
1565
|
+
}
|
|
1566
|
+
const ranges = [];
|
|
1567
|
+
for (const range of ipRanges) {
|
|
1568
|
+
if (!isObject2(range) || !isStringArray2(range.ipv4) || !isStringArray2(range.ipv6)) {
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
ranges.push({ ipv4: range.ipv4, ipv6: range.ipv6 });
|
|
1572
|
+
}
|
|
1573
|
+
return {
|
|
1574
|
+
tokens: tokens.map((token) => token.toLowerCase()),
|
|
1575
|
+
ipRanges: ranges
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
var lastAttemptOk = false;
|
|
1579
|
+
var lastAttemptAt = Number.NEGATIVE_INFINITY;
|
|
1580
|
+
async function fetchRules(apiUrl) {
|
|
1581
|
+
const response = await fetch(`${apiUrl}/api/v2/agent-traffic/rules`, {
|
|
1582
|
+
signal: AbortSignal.timeout(settings.requestTimeoutMs)
|
|
1583
|
+
});
|
|
1584
|
+
if (!response.ok) {
|
|
1585
|
+
return false;
|
|
1586
|
+
}
|
|
1587
|
+
const body = await response.json();
|
|
1588
|
+
const rules = parseRemoteRules(isObject2(body) ? body.data : null);
|
|
1589
|
+
if (!rules) {
|
|
1590
|
+
return false;
|
|
1591
|
+
}
|
|
1592
|
+
applyRules(rules);
|
|
1593
|
+
return true;
|
|
1594
|
+
}
|
|
1595
|
+
function refreshRules(apiUrl) {
|
|
1596
|
+
const now = Date.now();
|
|
1597
|
+
const wait = lastAttemptOk ? settings.rulesRefreshMs : settings.rulesRetryMs;
|
|
1598
|
+
if (now - lastAttemptAt < wait) {
|
|
1599
|
+
return null;
|
|
1600
|
+
}
|
|
1601
|
+
lastAttemptAt = now;
|
|
1602
|
+
lastAttemptOk = false;
|
|
1603
|
+
return fetchRules(apiUrl).then(
|
|
1604
|
+
(ok) => {
|
|
1605
|
+
lastAttemptOk = ok;
|
|
1606
|
+
},
|
|
1607
|
+
() => void 0
|
|
1608
|
+
);
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1264
1611
|
// src/wait-until.ts
|
|
1265
1612
|
var REQUEST_CONTEXT = /* @__PURE__ */ Symbol.for("@vercel/request-context");
|
|
1266
1613
|
function vercelRequestContext() {
|
|
@@ -1343,18 +1690,33 @@ function toRelayRequest(input) {
|
|
|
1343
1690
|
};
|
|
1344
1691
|
}
|
|
1345
1692
|
|
|
1346
|
-
// src/version.ts
|
|
1347
|
-
var RELAY_VERSION = true ? "0.3.0" : "0.0.0-dev";
|
|
1348
|
-
|
|
1349
1693
|
// src/index.ts
|
|
1350
|
-
function isAgent(request) {
|
|
1694
|
+
function isAgent(request, options = {}) {
|
|
1695
|
+
startRulesRefresh(options);
|
|
1351
1696
|
try {
|
|
1352
1697
|
const { headers, ip } = toRelayRequest(request);
|
|
1353
|
-
return decide({
|
|
1698
|
+
return decide({
|
|
1699
|
+
headers: readCappedHeaders(headers),
|
|
1700
|
+
ip,
|
|
1701
|
+
useBundled: options.rules === "bundled"
|
|
1702
|
+
});
|
|
1354
1703
|
} catch {
|
|
1355
1704
|
return false;
|
|
1356
1705
|
}
|
|
1357
1706
|
}
|
|
1707
|
+
function startRulesRefresh(options) {
|
|
1708
|
+
try {
|
|
1709
|
+
if (options.rules === "bundled") {
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
const apiUrl = resolveApiUrl(options);
|
|
1713
|
+
const work = apiUrl ? refreshRules(apiUrl) : null;
|
|
1714
|
+
if (work) {
|
|
1715
|
+
schedule(work, options.waitUntil);
|
|
1716
|
+
}
|
|
1717
|
+
} catch {
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1358
1720
|
function recordAgentVisit(request, options = {}) {
|
|
1359
1721
|
try {
|
|
1360
1722
|
const key = readKey(options.key);
|
|
@@ -1421,6 +1783,61 @@ function recordAgentVisit(request, options = {}) {
|
|
|
1421
1783
|
return Promise.resolve();
|
|
1422
1784
|
}
|
|
1423
1785
|
}
|
|
1786
|
+
async function serveAgentPage(request, options = {}) {
|
|
1787
|
+
try {
|
|
1788
|
+
const key = readKey(options.key);
|
|
1789
|
+
if (!key) {
|
|
1790
|
+
return null;
|
|
1791
|
+
}
|
|
1792
|
+
const apiUrl = resolveApiUrl(options);
|
|
1793
|
+
if (!apiUrl || isCloudFrontRequest(request)) {
|
|
1794
|
+
return null;
|
|
1795
|
+
}
|
|
1796
|
+
const method = request.method.toUpperCase();
|
|
1797
|
+
if (method !== "GET" && method !== "HEAD") {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
if (request.headers.has("x-waniwani-render")) {
|
|
1801
|
+
return null;
|
|
1802
|
+
}
|
|
1803
|
+
const rules = await loadSiteRules(
|
|
1804
|
+
apiUrl,
|
|
1805
|
+
key.ingestToken,
|
|
1806
|
+
(work) => schedule(work, void 0)
|
|
1807
|
+
);
|
|
1808
|
+
if (!rules) {
|
|
1809
|
+
return null;
|
|
1810
|
+
}
|
|
1811
|
+
const url = new URL(request.url);
|
|
1812
|
+
const site = rules.sites.find((entry) => entry.hostname === url.hostname);
|
|
1813
|
+
if (!site?.agentPages.enabled) {
|
|
1814
|
+
return null;
|
|
1815
|
+
}
|
|
1816
|
+
const path = normalizePath(url.pathname);
|
|
1817
|
+
if (!matchesPath(path, site.agentPages.paths)) {
|
|
1818
|
+
return null;
|
|
1819
|
+
}
|
|
1820
|
+
const userAgent = (request.headers.get("user-agent") ?? "").toLowerCase();
|
|
1821
|
+
if (!rules.userAgentTokens.some((token) => userAgent.includes(token))) {
|
|
1822
|
+
return null;
|
|
1823
|
+
}
|
|
1824
|
+
const page = await fetchAgentPage(rules, url.hostname, path);
|
|
1825
|
+
if (!page) {
|
|
1826
|
+
return null;
|
|
1827
|
+
}
|
|
1828
|
+
return new Response(method === "HEAD" ? null : page.body, {
|
|
1829
|
+
status: 200,
|
|
1830
|
+
headers: {
|
|
1831
|
+
"content-type": page.contentType,
|
|
1832
|
+
"x-waniwani-agent-page": "hit",
|
|
1833
|
+
"cache-control": "private, no-store",
|
|
1834
|
+
vary: "user-agent, accept"
|
|
1835
|
+
}
|
|
1836
|
+
});
|
|
1837
|
+
} catch {
|
|
1838
|
+
return null;
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1424
1841
|
var warnedRegion = false;
|
|
1425
1842
|
function resolveApiUrl(options) {
|
|
1426
1843
|
if (options.apiUrl !== void 0) {
|
|
@@ -1458,6 +1875,7 @@ function schedule(work, waitUntil) {
|
|
|
1458
1875
|
}
|
|
1459
1876
|
export {
|
|
1460
1877
|
isAgent,
|
|
1461
|
-
recordAgentVisit
|
|
1878
|
+
recordAgentVisit,
|
|
1879
|
+
serveAgentPage
|
|
1462
1880
|
};
|
|
1463
1881
|
//# sourceMappingURL=index.js.map
|