@waniwani/relay 0.1.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -101
- package/dist/index.d.ts +40 -6
- package/dist/index.js +696 -183
- package/dist/index.js.map +1 -1
- package/package.json +6 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,213 @@
|
|
|
1
|
+
// src/settings.ts
|
|
2
|
+
var settings = {
|
|
3
|
+
apiUrls: {
|
|
4
|
+
us: "https://app.waniwani.ai",
|
|
5
|
+
eu: "https://eu.app.waniwani.ai"
|
|
6
|
+
},
|
|
7
|
+
requestTimeoutMs: 2e3,
|
|
8
|
+
cloudFrontTimeoutMs: 1e3,
|
|
9
|
+
retryDelayMs: 500,
|
|
10
|
+
rulesRefreshMs: 12 * 60 * 60 * 1e3,
|
|
11
|
+
rulesRetryMs: 5 * 60 * 1e3,
|
|
12
|
+
siteRulesTimeoutMs: 300,
|
|
13
|
+
siteRulesRefreshMs: 6e4,
|
|
14
|
+
siteRulesRetryMs: 5e3
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
// src/version.ts
|
|
18
|
+
var RELAY_VERSION = true ? "0.4.0" : "0.0.0-dev";
|
|
19
|
+
|
|
20
|
+
// src/agent-page.ts
|
|
21
|
+
var isObject = (value) => typeof value === "object" && value !== null;
|
|
22
|
+
var isStringArray = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
23
|
+
var isPositive = (value) => typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
24
|
+
function parseSiteRules(body) {
|
|
25
|
+
if (!isObject(body) || body.version !== 1) {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const { pagesUrl, userAgentTokens, deadlines, maxBodyBytes, sites } = body;
|
|
29
|
+
if (typeof pagesUrl !== "string" || !isStringArray(userAgentTokens) || !isObject(deadlines) || !isPositive(deadlines.headersMs) || !isPositive(deadlines.bodyMs) || !isPositive(maxBodyBytes) || !Array.isArray(sites)) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
const parsedSites = [];
|
|
33
|
+
for (const site of sites) {
|
|
34
|
+
if (!isObject(site) || typeof site.hostname !== "string" || !isObject(site.agentPages) || typeof site.agentPages.enabled !== "boolean" || !isStringArray(site.agentPages.paths)) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
parsedSites.push({
|
|
38
|
+
hostname: site.hostname.toLowerCase(),
|
|
39
|
+
agentPages: {
|
|
40
|
+
enabled: site.agentPages.enabled,
|
|
41
|
+
paths: site.agentPages.paths
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
version: 1,
|
|
47
|
+
pagesUrl: pagesUrl.replace(/\/+$/, ""),
|
|
48
|
+
userAgentTokens: userAgentTokens.filter((token) => token !== ""),
|
|
49
|
+
deadlines: { headersMs: deadlines.headersMs, bodyMs: deadlines.bodyMs },
|
|
50
|
+
maxBodyBytes,
|
|
51
|
+
sites: parsedSites
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
var cache = /* @__PURE__ */ new Map();
|
|
55
|
+
async function fetchSiteRules(url, timeoutMs) {
|
|
56
|
+
const response = await fetch(url, {
|
|
57
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
58
|
+
});
|
|
59
|
+
if (!response.ok) {
|
|
60
|
+
await response.body?.cancel().catch(() => {
|
|
61
|
+
});
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
const body = await response.json();
|
|
65
|
+
return parseSiteRules(isObject(body) ? body.data : null);
|
|
66
|
+
}
|
|
67
|
+
function startFetch(entry, url, timeoutMs) {
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
entry.lastAttemptAt = now;
|
|
70
|
+
entry.lastAttemptOk = false;
|
|
71
|
+
const work = fetchSiteRules(url, timeoutMs).catch(() => null).then((rules) => {
|
|
72
|
+
if (rules) {
|
|
73
|
+
entry.rules = rules;
|
|
74
|
+
entry.fetchedAt = Date.now();
|
|
75
|
+
entry.lastAttemptOk = true;
|
|
76
|
+
}
|
|
77
|
+
entry.inFlight = null;
|
|
78
|
+
return rules;
|
|
79
|
+
});
|
|
80
|
+
entry.inFlight = work;
|
|
81
|
+
return work;
|
|
82
|
+
}
|
|
83
|
+
async function loadSiteRules(apiUrl, ingestToken, background) {
|
|
84
|
+
const url = `${apiUrl}/api/v2/agent-traffic/serve/${ingestToken}/rules`;
|
|
85
|
+
let entry = cache.get(url);
|
|
86
|
+
if (!entry) {
|
|
87
|
+
entry = {
|
|
88
|
+
rules: null,
|
|
89
|
+
fetchedAt: 0,
|
|
90
|
+
lastAttemptAt: Number.NEGATIVE_INFINITY,
|
|
91
|
+
lastAttemptOk: false,
|
|
92
|
+
inFlight: null
|
|
93
|
+
};
|
|
94
|
+
cache.set(url, entry);
|
|
95
|
+
}
|
|
96
|
+
const now = Date.now();
|
|
97
|
+
if (entry.rules) {
|
|
98
|
+
const stale = now - entry.fetchedAt >= settings.siteRulesRefreshMs;
|
|
99
|
+
const retryDue = entry.lastAttemptOk || now - entry.lastAttemptAt >= settings.siteRulesRetryMs;
|
|
100
|
+
if (stale && retryDue && !entry.inFlight) {
|
|
101
|
+
background(startFetch(entry, url, settings.requestTimeoutMs));
|
|
102
|
+
}
|
|
103
|
+
return entry.rules;
|
|
104
|
+
}
|
|
105
|
+
if (entry.inFlight) {
|
|
106
|
+
return entry.inFlight;
|
|
107
|
+
}
|
|
108
|
+
if (now - entry.lastAttemptAt < settings.siteRulesRetryMs) {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
return startFetch(entry, url, settings.siteRulesTimeoutMs);
|
|
112
|
+
}
|
|
113
|
+
function normalizePath(pathname) {
|
|
114
|
+
const trimmed = pathname.replace(/\/+$/, "");
|
|
115
|
+
return trimmed === "" ? "/" : trimmed;
|
|
116
|
+
}
|
|
117
|
+
function matchesPath(path, patterns) {
|
|
118
|
+
return patterns.some((pattern) => {
|
|
119
|
+
if (pattern === "*" || pattern === "/*") {
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
if (pattern.endsWith("/*")) {
|
|
123
|
+
const folder = normalizePath(pattern.slice(0, -2));
|
|
124
|
+
return path === folder || path.startsWith(`${folder}/`);
|
|
125
|
+
}
|
|
126
|
+
return path === normalizePath(pattern);
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
var TIMEOUT = /* @__PURE__ */ Symbol("timeout");
|
|
130
|
+
function within(promise, ms) {
|
|
131
|
+
let timer;
|
|
132
|
+
const deadline = new Promise((resolve) => {
|
|
133
|
+
timer = setTimeout(() => resolve(TIMEOUT), ms);
|
|
134
|
+
});
|
|
135
|
+
return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
|
|
136
|
+
}
|
|
137
|
+
async function readCapped(body, deadline, maxBytes) {
|
|
138
|
+
const reader = body.getReader();
|
|
139
|
+
const chunks = [];
|
|
140
|
+
let size = 0;
|
|
141
|
+
try {
|
|
142
|
+
while (true) {
|
|
143
|
+
const result = await Promise.race([reader.read(), deadline]);
|
|
144
|
+
if (result === TIMEOUT) {
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
if (result.done) {
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
size += result.value.byteLength;
|
|
151
|
+
if (size > maxBytes) {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
chunks.push(result.value);
|
|
155
|
+
}
|
|
156
|
+
} finally {
|
|
157
|
+
reader.cancel().catch(() => {
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
const out = new Uint8Array(size);
|
|
161
|
+
let offset = 0;
|
|
162
|
+
for (const chunk of chunks) {
|
|
163
|
+
out.set(chunk, offset);
|
|
164
|
+
offset += chunk.byteLength;
|
|
165
|
+
}
|
|
166
|
+
return out;
|
|
167
|
+
}
|
|
168
|
+
async function fetchAgentPage(rules, hostname, path) {
|
|
169
|
+
const controller = new AbortController();
|
|
170
|
+
try {
|
|
171
|
+
const response = await within(
|
|
172
|
+
fetch(`${rules.pagesUrl}/${hostname}${path}`, {
|
|
173
|
+
headers: {
|
|
174
|
+
accept: "text/markdown, text/html;q=0.9",
|
|
175
|
+
"x-waniwani-relay": RELAY_VERSION
|
|
176
|
+
},
|
|
177
|
+
signal: controller.signal
|
|
178
|
+
}),
|
|
179
|
+
rules.deadlines.headersMs
|
|
180
|
+
);
|
|
181
|
+
if (response === TIMEOUT) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
185
|
+
if (response.status !== 200 || !contentType.toLowerCase().startsWith("text/") || !response.body) {
|
|
186
|
+
await response.body?.cancel().catch(() => {
|
|
187
|
+
});
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
let timer;
|
|
191
|
+
const deadline = new Promise((resolve) => {
|
|
192
|
+
timer = setTimeout(() => resolve(TIMEOUT), rules.deadlines.bodyMs);
|
|
193
|
+
});
|
|
194
|
+
try {
|
|
195
|
+
const body = await readCapped(
|
|
196
|
+
response.body,
|
|
197
|
+
deadline,
|
|
198
|
+
rules.maxBodyBytes
|
|
199
|
+
);
|
|
200
|
+
return body ? { body, contentType } : null;
|
|
201
|
+
} finally {
|
|
202
|
+
clearTimeout(timer);
|
|
203
|
+
}
|
|
204
|
+
} catch {
|
|
205
|
+
return null;
|
|
206
|
+
} finally {
|
|
207
|
+
controller.abort();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
1
211
|
// src/headers.ts
|
|
2
212
|
var HEADER_CAP = [
|
|
3
213
|
"user-agent",
|
|
@@ -64,20 +274,10 @@ function clientIp(headers) {
|
|
|
64
274
|
return ip ? ip : null;
|
|
65
275
|
}
|
|
66
276
|
|
|
67
|
-
// src/settings.ts
|
|
68
|
-
var settings = {
|
|
69
|
-
apiUrls: {
|
|
70
|
-
us: "https://app.waniwani.ai",
|
|
71
|
-
eu: "https://eu.app.waniwani.ai"
|
|
72
|
-
},
|
|
73
|
-
requestTimeoutMs: 2e3,
|
|
74
|
-
retryDelayMs: 500
|
|
75
|
-
};
|
|
76
|
-
|
|
77
277
|
// src/http.ts
|
|
78
|
-
async function fetchWithTimeout(url, init) {
|
|
278
|
+
async function fetchWithTimeout(url, init, timeoutMs = settings.requestTimeoutMs) {
|
|
79
279
|
const controller = new AbortController();
|
|
80
|
-
const timer = setTimeout(() => controller.abort(),
|
|
280
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
81
281
|
try {
|
|
82
282
|
const response = await fetch(url, { ...init, signal: controller.signal });
|
|
83
283
|
await response.body?.cancel().catch(() => {
|
|
@@ -95,16 +295,20 @@ var delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
95
295
|
var isSuccess = (outcome) => "status" in outcome && outcome.status >= 200 && outcome.status < 300;
|
|
96
296
|
var isRetryable = (outcome) => "failure" in outcome || outcome.status === 429 || outcome.status >= 500;
|
|
97
297
|
var describe = (outcome) => "failure" in outcome ? outcome.failure : `status ${outcome.status}`;
|
|
98
|
-
var post = (url, body) => fetchWithTimeout(
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
298
|
+
var post = (url, body, timeoutMs) => fetchWithTimeout(
|
|
299
|
+
url,
|
|
300
|
+
{
|
|
301
|
+
method: "POST",
|
|
302
|
+
headers: { "content-type": "application/json" },
|
|
303
|
+
body
|
|
304
|
+
},
|
|
305
|
+
timeoutMs
|
|
306
|
+
);
|
|
307
|
+
async function postEvent(url, body, policy = { timeoutMs: settings.requestTimeoutMs, retry: true }) {
|
|
308
|
+
let outcome = await post(url, body, policy.timeoutMs);
|
|
309
|
+
if (policy.retry && !isSuccess(outcome) && isRetryable(outcome)) {
|
|
106
310
|
await delay(settings.retryDelayMs);
|
|
107
|
-
outcome = await post(url, body);
|
|
311
|
+
outcome = await post(url, body, policy.timeoutMs);
|
|
108
312
|
}
|
|
109
313
|
if (isSuccess(outcome)) {
|
|
110
314
|
return true;
|
|
@@ -146,165 +350,179 @@ function readEnv(name) {
|
|
|
146
350
|
// src/logic/decide.ts
|
|
147
351
|
import { BlockList } from "node:net";
|
|
148
352
|
|
|
149
|
-
// src/logic/agents.json
|
|
150
|
-
var
|
|
151
|
-
source: "
|
|
353
|
+
// src/logic/agents.generated.json
|
|
354
|
+
var agents_generated_default = {
|
|
355
|
+
source: "Generated by scripts/build-rules.ts from the WaniWani rules file d9b242ef451056c0, do not edit. Each token is matched case-insensitively inside the User-Agent header.",
|
|
152
356
|
tokens: [
|
|
153
|
-
"
|
|
154
|
-
"
|
|
155
|
-
"
|
|
156
|
-
"googleother",
|
|
157
|
-
"googleother-image",
|
|
158
|
-
"googleother-video",
|
|
159
|
-
"meta-externalagent",
|
|
160
|
-
"facebookbot",
|
|
161
|
-
"bytespider",
|
|
162
|
-
"tiktokspider",
|
|
163
|
-
"ccbot",
|
|
164
|
-
"diffbot",
|
|
357
|
+
"addsearchbot",
|
|
358
|
+
"agentdatabot",
|
|
359
|
+
"agenttimes",
|
|
165
360
|
"ai2bot",
|
|
361
|
+
"ai2bot-deepresearcheval",
|
|
166
362
|
"ai2bot-dolma",
|
|
167
|
-
"timpibot",
|
|
168
|
-
"cohere-training-data-crawler",
|
|
169
|
-
"mistralai-training",
|
|
170
|
-
"deepseekbot",
|
|
171
|
-
"erniebot",
|
|
172
|
-
"qwenbot",
|
|
173
|
-
"pangubot",
|
|
174
|
-
"kimibot",
|
|
175
|
-
"yandexadditional",
|
|
176
|
-
"yandexadditionalbot",
|
|
177
|
-
"brightbot",
|
|
178
|
-
"reflectionbot",
|
|
179
|
-
"agenttimes",
|
|
180
363
|
"aihitbot",
|
|
364
|
+
"aiwebindex",
|
|
365
|
+
"allenai.org",
|
|
366
|
+
"amazon-kendra",
|
|
367
|
+
"amazon-qbusiness",
|
|
368
|
+
"amazonbot",
|
|
369
|
+
"amazonbuyforme",
|
|
370
|
+
"amzn-searchbot",
|
|
371
|
+
"amzn-user",
|
|
372
|
+
"andibot",
|
|
373
|
+
"anthropic-ai",
|
|
374
|
+
"apifybot",
|
|
375
|
+
"apifywebsitecontentcrawler",
|
|
376
|
+
"applebot",
|
|
377
|
+
"aranet-searchbot",
|
|
378
|
+
"atlassian-bot",
|
|
181
379
|
"awario",
|
|
380
|
+
"azureai-searchbot",
|
|
381
|
+
"bedrockbot",
|
|
382
|
+
"bigsur.ai",
|
|
383
|
+
"bixelbot",
|
|
384
|
+
"bravebot",
|
|
385
|
+
"brightbot",
|
|
386
|
+
"buddybot",
|
|
387
|
+
"bytespider",
|
|
388
|
+
"ccbot",
|
|
389
|
+
"channel3bot",
|
|
182
390
|
"chatglm-spider",
|
|
391
|
+
"chatgpt agent",
|
|
392
|
+
"chatgpt-user",
|
|
393
|
+
"claude-code",
|
|
394
|
+
"claude-searchbot",
|
|
395
|
+
"claude-user",
|
|
396
|
+
"claude-web",
|
|
397
|
+
"claudebot",
|
|
398
|
+
"cloudflare-autorag",
|
|
399
|
+
"cloudflarebrowserrenderingcrawler",
|
|
400
|
+
"cloudvertexbot",
|
|
401
|
+
"cohere-ai",
|
|
402
|
+
"cohere-training-data-crawler",
|
|
183
403
|
"cotoyogi",
|
|
404
|
+
"coveobot",
|
|
405
|
+
"cragcrawler",
|
|
406
|
+
"crawl4ai",
|
|
407
|
+
"crawlspace",
|
|
408
|
+
"cursor/1.0.0",
|
|
184
409
|
"datenbank crawler",
|
|
410
|
+
"deepseekbot",
|
|
411
|
+
"diffbot",
|
|
412
|
+
"diffbot-user",
|
|
413
|
+
"doubaobot",
|
|
414
|
+
"duckassistbot",
|
|
185
415
|
"echobot bot",
|
|
186
416
|
"echoboxbot",
|
|
417
|
+
"erniebot",
|
|
418
|
+
"exabot",
|
|
419
|
+
"exasearchbot",
|
|
420
|
+
"exauser",
|
|
421
|
+
"facebookbot",
|
|
187
422
|
"factset_spyderbot",
|
|
423
|
+
"firecrawlagent",
|
|
188
424
|
"friendlycrawler",
|
|
425
|
+
"geisthaus-pagefetcher",
|
|
426
|
+
"gemini-deep-research",
|
|
427
|
+
"google-ads-creatives-assistant",
|
|
428
|
+
"google-agent",
|
|
429
|
+
"google-cloudvertexbot",
|
|
430
|
+
"google-firebase",
|
|
431
|
+
"google-gemini-cli",
|
|
432
|
+
"google-gemininotebook",
|
|
433
|
+
"google-lens",
|
|
434
|
+
"google-notebooklm",
|
|
435
|
+
"googleagent-mariner",
|
|
436
|
+
"googleagent-urlcontext",
|
|
437
|
+
"googleother",
|
|
438
|
+
"googleother-image",
|
|
439
|
+
"googleother-video",
|
|
440
|
+
"gptbot",
|
|
441
|
+
"grove/1.0 rss feed service",
|
|
442
|
+
"henkbot",
|
|
443
|
+
"iaskbot",
|
|
444
|
+
"iaskspider",
|
|
189
445
|
"icc-crawler",
|
|
190
446
|
"imagesiftbot",
|
|
191
447
|
"imagespider",
|
|
192
448
|
"img2dataset",
|
|
193
449
|
"isscyberriskcrawler",
|
|
450
|
+
"jackjillcareersbot",
|
|
451
|
+
"kagi-fetcher",
|
|
194
452
|
"kangaroo bot",
|
|
453
|
+
"keenablebot",
|
|
454
|
+
"kernel searchbot",
|
|
455
|
+
"kimi-agent",
|
|
456
|
+
"kimi-searchbot",
|
|
457
|
+
"kimi-user",
|
|
458
|
+
"kimibot",
|
|
459
|
+
"klaviyoaibot",
|
|
195
460
|
"kunatocrawler",
|
|
196
461
|
"laion-huggingface-processor",
|
|
197
462
|
"laiondownloader",
|
|
463
|
+
"lightpanda",
|
|
464
|
+
"linerbot",
|
|
465
|
+
"linkupbot",
|
|
466
|
+
"manus-user",
|
|
467
|
+
"meta-externalagent",
|
|
468
|
+
"meta-externalfetcher",
|
|
469
|
+
"meta-webindexer",
|
|
470
|
+
"mistralai-index",
|
|
471
|
+
"mistralai-training",
|
|
472
|
+
"mistralai-user",
|
|
473
|
+
"momentic",
|
|
474
|
+
"mozilla-tabstack",
|
|
198
475
|
"mycentralaiscraperbot",
|
|
199
476
|
"nagetbot",
|
|
200
477
|
"netestate imprint crawler",
|
|
478
|
+
"notebooklm",
|
|
479
|
+
"novaact",
|
|
480
|
+
"oai-adsbot",
|
|
481
|
+
"oai-searchbot",
|
|
201
482
|
"omgili",
|
|
202
483
|
"omgilibot",
|
|
484
|
+
"pangubot",
|
|
203
485
|
"panscient",
|
|
204
|
-
"
|
|
205
|
-
"sbintuitionsbot",
|
|
206
|
-
"sidetrade indexer bot",
|
|
207
|
-
"thinkbot",
|
|
208
|
-
"velenpublicwebcrawler",
|
|
209
|
-
"wardbot",
|
|
210
|
-
"oai-searchbot",
|
|
211
|
-
"oai-adsbot",
|
|
212
|
-
"claude-searchbot",
|
|
486
|
+
"perplexity-user",
|
|
213
487
|
"perplexitybot",
|
|
214
|
-
"applebot",
|
|
215
|
-
"amazonbot",
|
|
216
|
-
"amzn-searchbot",
|
|
217
|
-
"amazon-kendra",
|
|
218
|
-
"meta-webindexer",
|
|
219
|
-
"mistralai-index",
|
|
220
|
-
"kimi-searchbot",
|
|
221
|
-
"azureai-searchbot",
|
|
222
488
|
"petalbot",
|
|
223
|
-
"
|
|
224
|
-
"
|
|
225
|
-
"
|
|
226
|
-
"
|
|
227
|
-
"
|
|
489
|
+
"phindbot",
|
|
490
|
+
"poggio-citations",
|
|
491
|
+
"poseidon research crawler",
|
|
492
|
+
"qodercli",
|
|
493
|
+
"qualifiedbot",
|
|
494
|
+
"querit-searchbot",
|
|
495
|
+
"queritbot",
|
|
496
|
+
"quillbot",
|
|
497
|
+
"qwenbot",
|
|
498
|
+
"reflectionbot",
|
|
499
|
+
"ryebot",
|
|
500
|
+
"sbintuitionsbot",
|
|
228
501
|
"seltzsitemapfetcher",
|
|
502
|
+
"semanticscholarbot",
|
|
503
|
+
"semrushbot-ocob",
|
|
504
|
+
"semrushbot-swa",
|
|
505
|
+
"shap-user",
|
|
229
506
|
"shapbot",
|
|
507
|
+
"sidetrade indexer bot",
|
|
230
508
|
"tavilybot",
|
|
231
|
-
"bravebot",
|
|
232
|
-
"cloudflare-autorag",
|
|
233
|
-
"jackjillcareersbot",
|
|
234
|
-
"addsearchbot",
|
|
235
|
-
"aiwebindex",
|
|
236
|
-
"andibot",
|
|
237
|
-
"aranet-searchbot",
|
|
238
|
-
"channel3bot",
|
|
239
|
-
"iaskbot",
|
|
240
|
-
"iaskspider",
|
|
241
|
-
"querit-searchbot",
|
|
242
|
-
"queritbot",
|
|
243
|
-
"terracotta",
|
|
244
509
|
"terra cotta",
|
|
245
|
-
"
|
|
246
|
-
"
|
|
247
|
-
"
|
|
248
|
-
"
|
|
249
|
-
"claude-web",
|
|
250
|
-
"claude-code",
|
|
251
|
-
"perplexity-user",
|
|
252
|
-
"mistralai-user",
|
|
253
|
-
"meta-externalfetcher",
|
|
254
|
-
"google-cloudvertexbot",
|
|
255
|
-
"cloudvertexbot",
|
|
256
|
-
"google-firebase",
|
|
257
|
-
"google-notebooklm",
|
|
258
|
-
"notebooklm",
|
|
259
|
-
"gemini-deep-research",
|
|
260
|
-
"google-agent",
|
|
261
|
-
"googleagent-mariner",
|
|
262
|
-
"googleagent-urlcontext",
|
|
263
|
-
"google-gemini-cli",
|
|
264
|
-
"duckassistbot",
|
|
265
|
-
"cohere-ai",
|
|
266
|
-
"amzn-user",
|
|
267
|
-
"amazonbuyforme",
|
|
268
|
-
"amazon-qbusiness",
|
|
269
|
-
"novaact",
|
|
270
|
-
"bedrockbot",
|
|
271
|
-
"diffbot-user",
|
|
272
|
-
"ai2bot-deepresearcheval",
|
|
273
|
-
"kimi-agent",
|
|
274
|
-
"kimi-user",
|
|
275
|
-
"manus-user",
|
|
276
|
-
"kagi-fetcher",
|
|
277
|
-
"phindbot",
|
|
510
|
+
"terracotta",
|
|
511
|
+
"thinkbot",
|
|
512
|
+
"tiktokspider",
|
|
513
|
+
"timpibot",
|
|
278
514
|
"tongyibot",
|
|
279
|
-
"
|
|
280
|
-
"doubaobot",
|
|
281
|
-
"shap-user",
|
|
282
|
-
"firecrawlagent",
|
|
283
|
-
"apifybot",
|
|
284
|
-
"apifywebsitecontentcrawler",
|
|
285
|
-
"cloudflarebrowserrenderingcrawler",
|
|
286
|
-
"crawlspace",
|
|
287
|
-
"mozilla-tabstack",
|
|
288
|
-
"klaviyoaibot",
|
|
289
|
-
"atlassian-bot",
|
|
290
|
-
"semrushbot-ocob",
|
|
291
|
-
"semrushbot-swa",
|
|
292
|
-
"quillbot",
|
|
293
|
-
"bigsur.ai",
|
|
294
|
-
"geisthaus-pagefetcher",
|
|
295
|
-
"poggio-citations",
|
|
296
|
-
"qualifiedbot",
|
|
515
|
+
"twilioknowledge",
|
|
297
516
|
"twinagent",
|
|
298
|
-
"
|
|
299
|
-
"
|
|
300
|
-
"
|
|
301
|
-
"
|
|
302
|
-
"
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
"
|
|
306
|
-
"
|
|
307
|
-
"wrtnbot"
|
|
517
|
+
"v0bot",
|
|
518
|
+
"velenpublicwebcrawler",
|
|
519
|
+
"wardbot",
|
|
520
|
+
"wrtnbot",
|
|
521
|
+
"yandexadditional",
|
|
522
|
+
"yandexadditionalbot",
|
|
523
|
+
"yiyanbot",
|
|
524
|
+
"youbot",
|
|
525
|
+
"zanistabot"
|
|
308
526
|
]
|
|
309
527
|
};
|
|
310
528
|
|
|
@@ -624,6 +842,27 @@ var ip_ranges_generated_default = {
|
|
|
624
842
|
],
|
|
625
843
|
ipv6: []
|
|
626
844
|
},
|
|
845
|
+
{
|
|
846
|
+
bot: "ClaudeBot",
|
|
847
|
+
url: "https://claude.com/crawling/bots.json",
|
|
848
|
+
ipv4: [
|
|
849
|
+
"34.152.124.140/32",
|
|
850
|
+
"34.153.21.190/32",
|
|
851
|
+
"34.162.27.34/32",
|
|
852
|
+
"34.162.86.102/32",
|
|
853
|
+
"34.162.90.102/32",
|
|
854
|
+
"34.162.111.78/32",
|
|
855
|
+
"34.162.153.71/32",
|
|
856
|
+
"34.162.191.81/32",
|
|
857
|
+
"34.162.230.222/32",
|
|
858
|
+
"34.162.244.71/32",
|
|
859
|
+
"34.186.252.4/32",
|
|
860
|
+
"136.79.143.251/32",
|
|
861
|
+
"136.83.20.215/32",
|
|
862
|
+
"216.73.216.0/22"
|
|
863
|
+
],
|
|
864
|
+
ipv6: []
|
|
865
|
+
},
|
|
627
866
|
{
|
|
628
867
|
bot: "PerplexityBot",
|
|
629
868
|
url: "https://www.perplexity.com/perplexitybot.json",
|
|
@@ -1177,6 +1416,46 @@ var ip_ranges_generated_default = {
|
|
|
1177
1416
|
"191.235.202.48/32"
|
|
1178
1417
|
],
|
|
1179
1418
|
ipv6: []
|
|
1419
|
+
},
|
|
1420
|
+
{
|
|
1421
|
+
bot: "Applebot",
|
|
1422
|
+
url: "https://search.developer.apple.com/applebot.json",
|
|
1423
|
+
ipv4: [
|
|
1424
|
+
"17.166.20.0/24",
|
|
1425
|
+
"17.166.21.0/24",
|
|
1426
|
+
"17.166.22.0/24",
|
|
1427
|
+
"17.166.23.0/24",
|
|
1428
|
+
"17.166.24.0/24",
|
|
1429
|
+
"17.166.25.0/24",
|
|
1430
|
+
"17.166.150.0/24",
|
|
1431
|
+
"17.166.151.0/24",
|
|
1432
|
+
"17.166.152.0/24",
|
|
1433
|
+
"17.166.153.0/24",
|
|
1434
|
+
"17.166.154.0/24",
|
|
1435
|
+
"17.166.155.0/24",
|
|
1436
|
+
"17.166.192.48/28",
|
|
1437
|
+
"17.166.232.0/24",
|
|
1438
|
+
"17.166.233.0/24",
|
|
1439
|
+
"17.166.234.0/24",
|
|
1440
|
+
"17.166.235.0/24",
|
|
1441
|
+
"17.166.236.0/24",
|
|
1442
|
+
"17.166.237.0/24",
|
|
1443
|
+
"17.166.240.48/28",
|
|
1444
|
+
"17.166.248.48/28",
|
|
1445
|
+
"17.241.193.160/27",
|
|
1446
|
+
"17.241.200.160/27",
|
|
1447
|
+
"17.241.208.160/27"
|
|
1448
|
+
],
|
|
1449
|
+
ipv6: []
|
|
1450
|
+
},
|
|
1451
|
+
{
|
|
1452
|
+
bot: "Claude API",
|
|
1453
|
+
url: "https://platform.claude.com/docs/en/api/ip-addresses.md",
|
|
1454
|
+
ipv4: [
|
|
1455
|
+
"153.61.208.0/20",
|
|
1456
|
+
"160.79.104.0/21"
|
|
1457
|
+
],
|
|
1458
|
+
ipv6: []
|
|
1180
1459
|
}
|
|
1181
1460
|
]
|
|
1182
1461
|
};
|
|
@@ -1184,29 +1463,42 @@ var ip_ranges_generated_default = {
|
|
|
1184
1463
|
// src/logic/decide.ts
|
|
1185
1464
|
var MAX_SCANNED_USER_AGENT = 1024;
|
|
1186
1465
|
var PREFIX = 3;
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
const
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
list
|
|
1193
|
-
|
|
1194
|
-
|
|
1466
|
+
function compile(rules) {
|
|
1467
|
+
const tokensByPrefix = /* @__PURE__ */ new Map();
|
|
1468
|
+
for (const token of rules.tokens) {
|
|
1469
|
+
const key = token.slice(0, PREFIX);
|
|
1470
|
+
const list = tokensByPrefix.get(key);
|
|
1471
|
+
if (list) {
|
|
1472
|
+
list.push(token);
|
|
1473
|
+
} else {
|
|
1474
|
+
tokensByPrefix.set(key, [token]);
|
|
1475
|
+
}
|
|
1195
1476
|
}
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1477
|
+
const agentRanges = new BlockList();
|
|
1478
|
+
const addRanges = (cidrs, type) => {
|
|
1479
|
+
for (const cidr of cidrs) {
|
|
1480
|
+
const [address = "", prefix, extra] = cidr.split("/");
|
|
1481
|
+
if (!prefix || extra !== void 0) {
|
|
1482
|
+
throw new Error(`malformed range ${cidr}`);
|
|
1483
|
+
}
|
|
1484
|
+
agentRanges.addSubnet(address, Number(prefix), type);
|
|
1485
|
+
}
|
|
1486
|
+
};
|
|
1487
|
+
for (const source of rules.ipRanges) {
|
|
1488
|
+
addRanges(source.ipv4, "ipv4");
|
|
1489
|
+
addRanges(source.ipv6, "ipv6");
|
|
1202
1490
|
}
|
|
1203
|
-
};
|
|
1204
|
-
for (const source of ip_ranges_generated_default.sources) {
|
|
1205
|
-
addRanges(source.ipv4, "ipv4");
|
|
1206
|
-
addRanges(source.ipv6, "ipv6");
|
|
1491
|
+
return { tokensByPrefix, agentRanges };
|
|
1207
1492
|
}
|
|
1208
|
-
var
|
|
1209
|
-
|
|
1493
|
+
var bundled = compile({ tokens: agents_generated_default.tokens, ipRanges: ip_ranges_generated_default.sources });
|
|
1494
|
+
var remote = null;
|
|
1495
|
+
function applyRules(rules) {
|
|
1496
|
+
remote = compile(rules);
|
|
1497
|
+
}
|
|
1498
|
+
var pick = (useBundled) => useBundled ? bundled : remote ?? bundled;
|
|
1499
|
+
var isAgentIp = (ip, useBundled = false) => pick(useBundled).agentRanges.check(ip, ip.includes(":") ? "ipv6" : "ipv4");
|
|
1500
|
+
function hasAgentToken(userAgent, useBundled = false) {
|
|
1501
|
+
const { tokensByPrefix } = pick(useBundled);
|
|
1210
1502
|
const text = userAgent.slice(0, MAX_SCANNED_USER_AGENT).toLowerCase();
|
|
1211
1503
|
for (let i = 0; i + PREFIX <= text.length; i++) {
|
|
1212
1504
|
const candidates = tokensByPrefix.get(text.slice(i, i + PREFIX));
|
|
@@ -1220,9 +1512,13 @@ function hasAgentToken(userAgent) {
|
|
|
1220
1512
|
}
|
|
1221
1513
|
return false;
|
|
1222
1514
|
}
|
|
1223
|
-
function decide({
|
|
1515
|
+
function decide({
|
|
1516
|
+
headers,
|
|
1517
|
+
ip,
|
|
1518
|
+
useBundled = false
|
|
1519
|
+
}) {
|
|
1224
1520
|
const userAgent = headers["user-agent"];
|
|
1225
|
-
return userAgent !== void 0 && hasAgentToken(userAgent) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip);
|
|
1521
|
+
return userAgent !== void 0 && hasAgentToken(userAgent, useBundled) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip, useBundled);
|
|
1226
1522
|
}
|
|
1227
1523
|
|
|
1228
1524
|
// src/match-key.ts
|
|
@@ -1256,52 +1552,203 @@ async function deriveMatchKey(orgKey, ip) {
|
|
|
1256
1552
|
return toHex(signature);
|
|
1257
1553
|
}
|
|
1258
1554
|
|
|
1259
|
-
// src/
|
|
1260
|
-
var
|
|
1555
|
+
// src/refresh.ts
|
|
1556
|
+
var isObject2 = (value) => typeof value === "object" && value !== null;
|
|
1557
|
+
var isStringArray2 = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
1558
|
+
function parseRemoteRules(body) {
|
|
1559
|
+
if (!isObject2(body) || body.version !== 1) {
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1562
|
+
const { tokens, ipRanges } = body;
|
|
1563
|
+
if (!isStringArray2(tokens) || !Array.isArray(ipRanges)) {
|
|
1564
|
+
return null;
|
|
1565
|
+
}
|
|
1566
|
+
const ranges = [];
|
|
1567
|
+
for (const range of ipRanges) {
|
|
1568
|
+
if (!isObject2(range) || !isStringArray2(range.ipv4) || !isStringArray2(range.ipv6)) {
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
ranges.push({ ipv4: range.ipv4, ipv6: range.ipv6 });
|
|
1572
|
+
}
|
|
1573
|
+
return {
|
|
1574
|
+
tokens: tokens.map((token) => token.toLowerCase()),
|
|
1575
|
+
ipRanges: ranges
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
var lastAttemptOk = false;
|
|
1579
|
+
var lastAttemptAt = Number.NEGATIVE_INFINITY;
|
|
1580
|
+
async function fetchRules(apiUrl) {
|
|
1581
|
+
const response = await fetch(`${apiUrl}/api/v2/agent-traffic/rules`, {
|
|
1582
|
+
signal: AbortSignal.timeout(settings.requestTimeoutMs)
|
|
1583
|
+
});
|
|
1584
|
+
if (!response.ok) {
|
|
1585
|
+
return false;
|
|
1586
|
+
}
|
|
1587
|
+
const body = await response.json();
|
|
1588
|
+
const rules = parseRemoteRules(isObject2(body) ? body.data : null);
|
|
1589
|
+
if (!rules) {
|
|
1590
|
+
return false;
|
|
1591
|
+
}
|
|
1592
|
+
applyRules(rules);
|
|
1593
|
+
return true;
|
|
1594
|
+
}
|
|
1595
|
+
function refreshRules(apiUrl) {
|
|
1596
|
+
const now = Date.now();
|
|
1597
|
+
const wait = lastAttemptOk ? settings.rulesRefreshMs : settings.rulesRetryMs;
|
|
1598
|
+
if (now - lastAttemptAt < wait) {
|
|
1599
|
+
return null;
|
|
1600
|
+
}
|
|
1601
|
+
lastAttemptAt = now;
|
|
1602
|
+
lastAttemptOk = false;
|
|
1603
|
+
return fetchRules(apiUrl).then(
|
|
1604
|
+
(ok) => {
|
|
1605
|
+
lastAttemptOk = ok;
|
|
1606
|
+
},
|
|
1607
|
+
() => void 0
|
|
1608
|
+
);
|
|
1609
|
+
}
|
|
1261
1610
|
|
|
1262
1611
|
// src/wait-until.ts
|
|
1263
1612
|
var REQUEST_CONTEXT = /* @__PURE__ */ Symbol.for("@vercel/request-context");
|
|
1264
|
-
function
|
|
1613
|
+
function vercelRequestContext() {
|
|
1265
1614
|
const store = Reflect.get(
|
|
1266
1615
|
globalThis,
|
|
1267
1616
|
REQUEST_CONTEXT
|
|
1268
1617
|
);
|
|
1269
|
-
|
|
1618
|
+
return store?.get?.();
|
|
1619
|
+
}
|
|
1620
|
+
function vercelWaitUntil() {
|
|
1621
|
+
const context = vercelRequestContext();
|
|
1270
1622
|
return context?.waitUntil ? (promise) => context.waitUntil?.(promise) : void 0;
|
|
1271
1623
|
}
|
|
1272
1624
|
|
|
1625
|
+
// src/platform.ts
|
|
1626
|
+
function detectPlatform() {
|
|
1627
|
+
try {
|
|
1628
|
+
const context = vercelRequestContext();
|
|
1629
|
+
if (typeof context === "object" && context !== null) {
|
|
1630
|
+
return "vercel";
|
|
1631
|
+
}
|
|
1632
|
+
if (globalThis.navigator?.userAgent === "Cloudflare-Workers") {
|
|
1633
|
+
return "cloudflare";
|
|
1634
|
+
}
|
|
1635
|
+
} catch {
|
|
1636
|
+
}
|
|
1637
|
+
return void 0;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
// src/request.ts
|
|
1641
|
+
function isCloudFrontRequest(input) {
|
|
1642
|
+
return typeof Reflect.get(input, "clientIp") === "string" && typeof Reflect.get(input, "uri") === "string";
|
|
1643
|
+
}
|
|
1644
|
+
function originOf(host) {
|
|
1645
|
+
if (!host) {
|
|
1646
|
+
return null;
|
|
1647
|
+
}
|
|
1648
|
+
try {
|
|
1649
|
+
const url = new URL(`https://${host}`);
|
|
1650
|
+
const bare = url.pathname === "/" && url.search === "" && url.hash === "" && url.username === "" && url.password === "";
|
|
1651
|
+
return bare ? url.origin : null;
|
|
1652
|
+
} catch {
|
|
1653
|
+
return null;
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
function cloudFrontHeaders(input) {
|
|
1657
|
+
const headers = new Headers();
|
|
1658
|
+
for (const name of HEADER_CAP) {
|
|
1659
|
+
for (const entry of input?.[name] ?? []) {
|
|
1660
|
+
if (typeof entry?.value !== "string") {
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
try {
|
|
1664
|
+
headers.append(name, entry.value);
|
|
1665
|
+
} catch {
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
return headers;
|
|
1670
|
+
}
|
|
1671
|
+
function toRelayRequest(input) {
|
|
1672
|
+
if (!isCloudFrontRequest(input)) {
|
|
1673
|
+
return {
|
|
1674
|
+
url: input.url,
|
|
1675
|
+
method: input.method,
|
|
1676
|
+
headers: input.headers,
|
|
1677
|
+
ip: clientIp(input.headers),
|
|
1678
|
+
platform: detectPlatform()
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
1681
|
+
const origin = originOf(input.headers?.host?.[0]?.value);
|
|
1682
|
+
const query = input.querystring ? `?${input.querystring}` : "";
|
|
1683
|
+
const ip = input.clientIp.trim().toLowerCase();
|
|
1684
|
+
return {
|
|
1685
|
+
url: origin === null ? null : `${origin}${input.uri}${query}`,
|
|
1686
|
+
method: input.method,
|
|
1687
|
+
headers: cloudFrontHeaders(input.headers),
|
|
1688
|
+
ip: ip ? ip : null,
|
|
1689
|
+
platform: "cloudfront"
|
|
1690
|
+
};
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1273
1693
|
// src/index.ts
|
|
1274
|
-
function isAgent(request) {
|
|
1694
|
+
function isAgent(request, options = {}) {
|
|
1695
|
+
startRulesRefresh(options);
|
|
1275
1696
|
try {
|
|
1276
|
-
const headers =
|
|
1277
|
-
return decide({
|
|
1697
|
+
const { headers, ip } = toRelayRequest(request);
|
|
1698
|
+
return decide({
|
|
1699
|
+
headers: readCappedHeaders(headers),
|
|
1700
|
+
ip,
|
|
1701
|
+
useBundled: options.rules === "bundled"
|
|
1702
|
+
});
|
|
1278
1703
|
} catch {
|
|
1279
1704
|
return false;
|
|
1280
1705
|
}
|
|
1281
1706
|
}
|
|
1707
|
+
function startRulesRefresh(options) {
|
|
1708
|
+
try {
|
|
1709
|
+
if (options.rules === "bundled") {
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
const apiUrl = resolveApiUrl(options);
|
|
1713
|
+
const work = apiUrl ? refreshRules(apiUrl) : null;
|
|
1714
|
+
if (work) {
|
|
1715
|
+
schedule(work, options.waitUntil);
|
|
1716
|
+
}
|
|
1717
|
+
} catch {
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1282
1720
|
function recordAgentVisit(request, options = {}) {
|
|
1283
1721
|
try {
|
|
1284
1722
|
const key = readKey(options.key);
|
|
1285
1723
|
if (!key) {
|
|
1286
|
-
return;
|
|
1724
|
+
return Promise.resolve();
|
|
1287
1725
|
}
|
|
1288
1726
|
const apiUrl = resolveApiUrl(options);
|
|
1289
1727
|
if (!apiUrl) {
|
|
1290
|
-
return;
|
|
1728
|
+
return Promise.resolve();
|
|
1291
1729
|
}
|
|
1292
1730
|
const omitted = omitSet(options.omit);
|
|
1293
|
-
const
|
|
1294
|
-
|
|
1731
|
+
const {
|
|
1732
|
+
url: requestUrl,
|
|
1733
|
+
method,
|
|
1734
|
+
headers: requestHeaders,
|
|
1735
|
+
ip,
|
|
1736
|
+
platform
|
|
1737
|
+
} = toRelayRequest(request);
|
|
1738
|
+
if (requestUrl === null) {
|
|
1739
|
+
return Promise.resolve();
|
|
1740
|
+
}
|
|
1741
|
+
const capped = readCappedHeaders(requestHeaders);
|
|
1295
1742
|
const headers = {};
|
|
1296
1743
|
for (const [name, value] of Object.entries(capped)) {
|
|
1297
1744
|
if (!omitted.has(name)) {
|
|
1298
1745
|
headers[name] = value;
|
|
1299
1746
|
}
|
|
1300
1747
|
}
|
|
1301
|
-
const url = new URL(
|
|
1748
|
+
const url = new URL(requestUrl);
|
|
1302
1749
|
const record = {
|
|
1303
1750
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1304
|
-
method
|
|
1751
|
+
method,
|
|
1305
1752
|
host: url.hostname,
|
|
1306
1753
|
path: url.pathname + url.search,
|
|
1307
1754
|
headers,
|
|
@@ -1314,6 +1761,10 @@ function recordAgentVisit(request, options = {}) {
|
|
|
1314
1761
|
if (capped.referer !== void 0 && !omitted.has("referer")) {
|
|
1315
1762
|
record.referer = capped.referer;
|
|
1316
1763
|
}
|
|
1764
|
+
if (platform !== void 0) {
|
|
1765
|
+
record.platform = platform;
|
|
1766
|
+
}
|
|
1767
|
+
const policy = platform === "cloudfront" ? { timeoutMs: settings.cloudFrontTimeoutMs, retry: false } : { timeoutMs: settings.requestTimeoutMs, retry: true };
|
|
1317
1768
|
const ingestUrl = `${apiUrl}/api/v2/agent-traffic/ingest/middleware/${key.ingestToken}`;
|
|
1318
1769
|
const withMatchKey = async () => {
|
|
1319
1770
|
if (ip !== null && !omitted.has("matchkey")) {
|
|
@@ -1321,10 +1772,70 @@ function recordAgentVisit(request, options = {}) {
|
|
|
1321
1772
|
() => void 0
|
|
1322
1773
|
);
|
|
1323
1774
|
}
|
|
1324
|
-
await postEvent(
|
|
1775
|
+
await postEvent(
|
|
1776
|
+
ingestUrl,
|
|
1777
|
+
JSON.stringify({ requests: [record] }),
|
|
1778
|
+
policy
|
|
1779
|
+
);
|
|
1325
1780
|
};
|
|
1326
|
-
schedule(withMatchKey(), options.waitUntil);
|
|
1781
|
+
return schedule(withMatchKey(), options.waitUntil);
|
|
1782
|
+
} catch {
|
|
1783
|
+
return Promise.resolve();
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
async function serveAgentPage(request, options = {}) {
|
|
1787
|
+
try {
|
|
1788
|
+
const key = readKey(options.key);
|
|
1789
|
+
if (!key) {
|
|
1790
|
+
return null;
|
|
1791
|
+
}
|
|
1792
|
+
const apiUrl = resolveApiUrl(options);
|
|
1793
|
+
if (!apiUrl || isCloudFrontRequest(request)) {
|
|
1794
|
+
return null;
|
|
1795
|
+
}
|
|
1796
|
+
const method = request.method.toUpperCase();
|
|
1797
|
+
if (method !== "GET" && method !== "HEAD") {
|
|
1798
|
+
return null;
|
|
1799
|
+
}
|
|
1800
|
+
if (request.headers.has("x-waniwani-render")) {
|
|
1801
|
+
return null;
|
|
1802
|
+
}
|
|
1803
|
+
const rules = await loadSiteRules(
|
|
1804
|
+
apiUrl,
|
|
1805
|
+
key.ingestToken,
|
|
1806
|
+
(work) => schedule(work, void 0)
|
|
1807
|
+
);
|
|
1808
|
+
if (!rules) {
|
|
1809
|
+
return null;
|
|
1810
|
+
}
|
|
1811
|
+
const url = new URL(request.url);
|
|
1812
|
+
const site = rules.sites.find((entry) => entry.hostname === url.hostname);
|
|
1813
|
+
if (!site?.agentPages.enabled) {
|
|
1814
|
+
return null;
|
|
1815
|
+
}
|
|
1816
|
+
const path = normalizePath(url.pathname);
|
|
1817
|
+
if (!matchesPath(path, site.agentPages.paths)) {
|
|
1818
|
+
return null;
|
|
1819
|
+
}
|
|
1820
|
+
const userAgent = (request.headers.get("user-agent") ?? "").toLowerCase();
|
|
1821
|
+
if (!rules.userAgentTokens.some((token) => userAgent.includes(token))) {
|
|
1822
|
+
return null;
|
|
1823
|
+
}
|
|
1824
|
+
const page = await fetchAgentPage(rules, url.hostname, path);
|
|
1825
|
+
if (!page) {
|
|
1826
|
+
return null;
|
|
1827
|
+
}
|
|
1828
|
+
return new Response(method === "HEAD" ? null : page.body, {
|
|
1829
|
+
status: 200,
|
|
1830
|
+
headers: {
|
|
1831
|
+
"content-type": page.contentType,
|
|
1832
|
+
"x-waniwani-agent-page": "hit",
|
|
1833
|
+
"cache-control": "private, no-store",
|
|
1834
|
+
vary: "user-agent, accept"
|
|
1835
|
+
}
|
|
1836
|
+
});
|
|
1327
1837
|
} catch {
|
|
1838
|
+
return null;
|
|
1328
1839
|
}
|
|
1329
1840
|
}
|
|
1330
1841
|
var warnedRegion = false;
|
|
@@ -1360,9 +1871,11 @@ function schedule(work, waitUntil) {
|
|
|
1360
1871
|
(waitUntil ?? vercelWaitUntil())?.(settled);
|
|
1361
1872
|
} catch {
|
|
1362
1873
|
}
|
|
1874
|
+
return settled;
|
|
1363
1875
|
}
|
|
1364
1876
|
export {
|
|
1365
1877
|
isAgent,
|
|
1366
|
-
recordAgentVisit
|
|
1878
|
+
recordAgentVisit,
|
|
1879
|
+
serveAgentPage
|
|
1367
1880
|
};
|
|
1368
1881
|
//# sourceMappingURL=index.js.map
|