@waniwani/relay 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -6
- package/dist/index.d.ts +42 -4
- package/dist/index.js +753 -163
- package/dist/index.js.map +1 -1
- package/package.json +3 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,221 @@
|
|
|
1
|
+
// src/settings.ts
|
|
2
|
+
var settings = {
|
|
3
|
+
apiUrls: {
|
|
4
|
+
us: "https://app.waniwani.ai",
|
|
5
|
+
eu: "https://eu.app.waniwani.ai"
|
|
6
|
+
},
|
|
7
|
+
requestTimeoutMs: 2e3,
|
|
8
|
+
cloudFrontTimeoutMs: 1e3,
|
|
9
|
+
cloudFrontMaxUriBytes: 8192,
|
|
10
|
+
cloudFrontMaxBodyBytes: 1e6 - 16384,
|
|
11
|
+
retryDelayMs: 500,
|
|
12
|
+
rulesRefreshMs: 12 * 60 * 60 * 1e3,
|
|
13
|
+
rulesRetryMs: 5 * 60 * 1e3,
|
|
14
|
+
siteRulesTimeoutMs: 300,
|
|
15
|
+
siteRulesRefreshMs: 6e4,
|
|
16
|
+
siteRulesRetryMs: 5e3
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
// src/version.ts
|
|
20
|
+
var RELAY_VERSION = true ? "0.4.1" : "0.0.0-dev";
|
|
21
|
+
|
|
22
|
+
// src/agent-page.ts
|
|
23
|
+
var isObject = (value) => typeof value === "object" && value !== null;
|
|
24
|
+
var isStringArray = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
25
|
+
var isPositive = (value) => typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
26
|
+
var cleanTokens = (tokens) => tokens.map((token) => token.trim().toLowerCase()).filter((token) => token !== "");
|
|
27
|
+
function parseSiteRules(body) {
|
|
28
|
+
if (!isObject(body) || body.version !== 1) {
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
const { pagesUrl, userAgentTokens, deadlines, maxBodyBytes, sites } = body;
|
|
32
|
+
if (typeof pagesUrl !== "string" || !isStringArray(userAgentTokens) || !isObject(deadlines) || !isPositive(deadlines.headersMs) || !isPositive(deadlines.bodyMs) || !isPositive(maxBodyBytes) || !Array.isArray(sites)) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
const parsedSites = [];
|
|
36
|
+
for (const site of sites) {
|
|
37
|
+
if (!isObject(site)) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
const blocked = site.blockedUserAgentTokens ?? [];
|
|
41
|
+
if (typeof site.hostname !== "string" || !isStringArray(blocked) || !isObject(site.agentPages) || typeof site.agentPages.enabled !== "boolean" || !isStringArray(site.agentPages.paths)) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
parsedSites.push({
|
|
45
|
+
hostname: site.hostname.toLowerCase(),
|
|
46
|
+
blockedUserAgentTokens: cleanTokens(blocked),
|
|
47
|
+
agentPages: {
|
|
48
|
+
enabled: site.agentPages.enabled,
|
|
49
|
+
paths: site.agentPages.paths
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
version: 1,
|
|
55
|
+
pagesUrl: pagesUrl.replace(/\/+$/, ""),
|
|
56
|
+
userAgentTokens: cleanTokens(userAgentTokens),
|
|
57
|
+
deadlines: { headersMs: deadlines.headersMs, bodyMs: deadlines.bodyMs },
|
|
58
|
+
maxBodyBytes,
|
|
59
|
+
sites: parsedSites
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
var cache = /* @__PURE__ */ new Map();
|
|
63
|
+
async function fetchSiteRules(url, timeoutMs) {
|
|
64
|
+
const response = await fetch(url, {
|
|
65
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
66
|
+
});
|
|
67
|
+
if (!response.ok) {
|
|
68
|
+
await response.body?.cancel().catch(() => {
|
|
69
|
+
});
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
const body = await response.json();
|
|
73
|
+
return parseSiteRules(isObject(body) ? body.data : null);
|
|
74
|
+
}
|
|
75
|
+
function startFetch(entry, url, timeoutMs) {
|
|
76
|
+
const now = Date.now();
|
|
77
|
+
entry.lastAttemptAt = now;
|
|
78
|
+
entry.lastAttemptOk = false;
|
|
79
|
+
const work = fetchSiteRules(url, timeoutMs).catch(() => null).then((rules) => {
|
|
80
|
+
if (rules) {
|
|
81
|
+
entry.rules = rules;
|
|
82
|
+
entry.fetchedAt = Date.now();
|
|
83
|
+
entry.lastAttemptOk = true;
|
|
84
|
+
}
|
|
85
|
+
entry.inFlight = null;
|
|
86
|
+
return rules;
|
|
87
|
+
});
|
|
88
|
+
entry.inFlight = work;
|
|
89
|
+
return work;
|
|
90
|
+
}
|
|
91
|
+
async function loadSiteRules(apiUrl, ingestToken, background) {
|
|
92
|
+
const url = `${apiUrl}/api/v2/agent-traffic/serve/${ingestToken}/rules`;
|
|
93
|
+
let entry = cache.get(url);
|
|
94
|
+
if (!entry) {
|
|
95
|
+
entry = {
|
|
96
|
+
rules: null,
|
|
97
|
+
fetchedAt: 0,
|
|
98
|
+
lastAttemptAt: Number.NEGATIVE_INFINITY,
|
|
99
|
+
lastAttemptOk: false,
|
|
100
|
+
inFlight: null
|
|
101
|
+
};
|
|
102
|
+
cache.set(url, entry);
|
|
103
|
+
}
|
|
104
|
+
const now = Date.now();
|
|
105
|
+
if (entry.rules) {
|
|
106
|
+
const stale = now - entry.fetchedAt >= settings.siteRulesRefreshMs;
|
|
107
|
+
const retryDue = entry.lastAttemptOk || now - entry.lastAttemptAt >= settings.siteRulesRetryMs;
|
|
108
|
+
if (stale && retryDue && !entry.inFlight) {
|
|
109
|
+
background(startFetch(entry, url, settings.requestTimeoutMs));
|
|
110
|
+
}
|
|
111
|
+
return entry.rules;
|
|
112
|
+
}
|
|
113
|
+
if (entry.inFlight) {
|
|
114
|
+
return entry.inFlight;
|
|
115
|
+
}
|
|
116
|
+
if (now - entry.lastAttemptAt < settings.siteRulesRetryMs) {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
return startFetch(entry, url, settings.siteRulesTimeoutMs);
|
|
120
|
+
}
|
|
121
|
+
function normalizePath(pathname) {
|
|
122
|
+
const trimmed = pathname.replace(/\/+$/, "");
|
|
123
|
+
return trimmed === "" ? "/" : trimmed;
|
|
124
|
+
}
|
|
125
|
+
function matchesPath(path, patterns) {
|
|
126
|
+
return patterns.some((pattern) => {
|
|
127
|
+
if (pattern === "*" || pattern === "/*") {
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
if (pattern.endsWith("/*")) {
|
|
131
|
+
const folder = normalizePath(pattern.slice(0, -2));
|
|
132
|
+
return path === folder || path.startsWith(`${folder}/`);
|
|
133
|
+
}
|
|
134
|
+
return path === normalizePath(pattern);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
var TIMEOUT = /* @__PURE__ */ Symbol("timeout");
|
|
138
|
+
function within(promise, ms) {
|
|
139
|
+
let timer;
|
|
140
|
+
const deadline = new Promise((resolve) => {
|
|
141
|
+
timer = setTimeout(() => resolve(TIMEOUT), ms);
|
|
142
|
+
});
|
|
143
|
+
return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
|
|
144
|
+
}
|
|
145
|
+
async function readCapped(body, deadline, maxBytes) {
|
|
146
|
+
const reader = body.getReader();
|
|
147
|
+
const chunks = [];
|
|
148
|
+
let size = 0;
|
|
149
|
+
try {
|
|
150
|
+
while (true) {
|
|
151
|
+
const result = await Promise.race([reader.read(), deadline]);
|
|
152
|
+
if (result === TIMEOUT) {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
if (result.done) {
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
size += result.value.byteLength;
|
|
159
|
+
if (size > maxBytes) {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
chunks.push(result.value);
|
|
163
|
+
}
|
|
164
|
+
} finally {
|
|
165
|
+
reader.cancel().catch(() => {
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
const out = new Uint8Array(size);
|
|
169
|
+
let offset = 0;
|
|
170
|
+
for (const chunk of chunks) {
|
|
171
|
+
out.set(chunk, offset);
|
|
172
|
+
offset += chunk.byteLength;
|
|
173
|
+
}
|
|
174
|
+
return out;
|
|
175
|
+
}
|
|
176
|
+
async function fetchAgentPage(rules, hostname, path) {
|
|
177
|
+
const controller = new AbortController();
|
|
178
|
+
try {
|
|
179
|
+
const response = await within(
|
|
180
|
+
fetch(`${rules.pagesUrl}/${hostname}${path}`, {
|
|
181
|
+
headers: {
|
|
182
|
+
accept: "text/markdown, text/html;q=0.9",
|
|
183
|
+
"x-waniwani-relay": RELAY_VERSION
|
|
184
|
+
},
|
|
185
|
+
signal: controller.signal
|
|
186
|
+
}),
|
|
187
|
+
rules.deadlines.headersMs
|
|
188
|
+
);
|
|
189
|
+
if (response === TIMEOUT) {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
193
|
+
if (response.status !== 200 || !contentType.toLowerCase().startsWith("text/") || !response.body) {
|
|
194
|
+
await response.body?.cancel().catch(() => {
|
|
195
|
+
});
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
let timer;
|
|
199
|
+
const deadline = new Promise((resolve) => {
|
|
200
|
+
timer = setTimeout(() => resolve(TIMEOUT), rules.deadlines.bodyMs);
|
|
201
|
+
});
|
|
202
|
+
try {
|
|
203
|
+
const body = await readCapped(
|
|
204
|
+
response.body,
|
|
205
|
+
deadline,
|
|
206
|
+
rules.maxBodyBytes
|
|
207
|
+
);
|
|
208
|
+
return body ? { body, contentType } : null;
|
|
209
|
+
} finally {
|
|
210
|
+
clearTimeout(timer);
|
|
211
|
+
}
|
|
212
|
+
} catch {
|
|
213
|
+
return null;
|
|
214
|
+
} finally {
|
|
215
|
+
controller.abort();
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
1
219
|
// src/headers.ts
|
|
2
220
|
var HEADER_CAP = [
|
|
3
221
|
"user-agent",
|
|
@@ -64,17 +282,6 @@ function clientIp(headers) {
|
|
|
64
282
|
return ip ? ip : null;
|
|
65
283
|
}
|
|
66
284
|
|
|
67
|
-
// src/settings.ts
|
|
68
|
-
var settings = {
|
|
69
|
-
apiUrls: {
|
|
70
|
-
us: "https://app.waniwani.ai",
|
|
71
|
-
eu: "https://eu.app.waniwani.ai"
|
|
72
|
-
},
|
|
73
|
-
requestTimeoutMs: 2e3,
|
|
74
|
-
cloudFrontTimeoutMs: 1e3,
|
|
75
|
-
retryDelayMs: 500
|
|
76
|
-
};
|
|
77
|
-
|
|
78
285
|
// src/http.ts
|
|
79
286
|
async function fetchWithTimeout(url, init, timeoutMs = settings.requestTimeoutMs) {
|
|
80
287
|
const controller = new AbortController();
|
|
@@ -151,165 +358,179 @@ function readEnv(name) {
|
|
|
151
358
|
// src/logic/decide.ts
|
|
152
359
|
import { BlockList } from "node:net";
|
|
153
360
|
|
|
154
|
-
// src/logic/agents.json
|
|
155
|
-
var
|
|
156
|
-
source: "
|
|
361
|
+
// src/logic/agents.generated.json
|
|
362
|
+
var agents_generated_default = {
|
|
363
|
+
source: "Generated by scripts/build-rules.ts from the WaniWani rules file d9b242ef451056c0, do not edit. Each token is matched case-insensitively inside the User-Agent header.",
|
|
157
364
|
tokens: [
|
|
158
|
-
"
|
|
159
|
-
"
|
|
160
|
-
"
|
|
161
|
-
"googleother",
|
|
162
|
-
"googleother-image",
|
|
163
|
-
"googleother-video",
|
|
164
|
-
"meta-externalagent",
|
|
165
|
-
"facebookbot",
|
|
166
|
-
"bytespider",
|
|
167
|
-
"tiktokspider",
|
|
168
|
-
"ccbot",
|
|
169
|
-
"diffbot",
|
|
365
|
+
"addsearchbot",
|
|
366
|
+
"agentdatabot",
|
|
367
|
+
"agenttimes",
|
|
170
368
|
"ai2bot",
|
|
369
|
+
"ai2bot-deepresearcheval",
|
|
171
370
|
"ai2bot-dolma",
|
|
172
|
-
"timpibot",
|
|
173
|
-
"cohere-training-data-crawler",
|
|
174
|
-
"mistralai-training",
|
|
175
|
-
"deepseekbot",
|
|
176
|
-
"erniebot",
|
|
177
|
-
"qwenbot",
|
|
178
|
-
"pangubot",
|
|
179
|
-
"kimibot",
|
|
180
|
-
"yandexadditional",
|
|
181
|
-
"yandexadditionalbot",
|
|
182
|
-
"brightbot",
|
|
183
|
-
"reflectionbot",
|
|
184
|
-
"agenttimes",
|
|
185
371
|
"aihitbot",
|
|
372
|
+
"aiwebindex",
|
|
373
|
+
"allenai.org",
|
|
374
|
+
"amazon-kendra",
|
|
375
|
+
"amazon-qbusiness",
|
|
376
|
+
"amazonbot",
|
|
377
|
+
"amazonbuyforme",
|
|
378
|
+
"amzn-searchbot",
|
|
379
|
+
"amzn-user",
|
|
380
|
+
"andibot",
|
|
381
|
+
"anthropic-ai",
|
|
382
|
+
"apifybot",
|
|
383
|
+
"apifywebsitecontentcrawler",
|
|
384
|
+
"applebot",
|
|
385
|
+
"aranet-searchbot",
|
|
386
|
+
"atlassian-bot",
|
|
186
387
|
"awario",
|
|
388
|
+
"azureai-searchbot",
|
|
389
|
+
"bedrockbot",
|
|
390
|
+
"bigsur.ai",
|
|
391
|
+
"bixelbot",
|
|
392
|
+
"bravebot",
|
|
393
|
+
"brightbot",
|
|
394
|
+
"buddybot",
|
|
395
|
+
"bytespider",
|
|
396
|
+
"ccbot",
|
|
397
|
+
"channel3bot",
|
|
187
398
|
"chatglm-spider",
|
|
399
|
+
"chatgpt agent",
|
|
400
|
+
"chatgpt-user",
|
|
401
|
+
"claude-code",
|
|
402
|
+
"claude-searchbot",
|
|
403
|
+
"claude-user",
|
|
404
|
+
"claude-web",
|
|
405
|
+
"claudebot",
|
|
406
|
+
"cloudflare-autorag",
|
|
407
|
+
"cloudflarebrowserrenderingcrawler",
|
|
408
|
+
"cloudvertexbot",
|
|
409
|
+
"cohere-ai",
|
|
410
|
+
"cohere-training-data-crawler",
|
|
188
411
|
"cotoyogi",
|
|
412
|
+
"coveobot",
|
|
413
|
+
"cragcrawler",
|
|
414
|
+
"crawl4ai",
|
|
415
|
+
"crawlspace",
|
|
416
|
+
"cursor/1.0.0",
|
|
189
417
|
"datenbank crawler",
|
|
418
|
+
"deepseekbot",
|
|
419
|
+
"diffbot",
|
|
420
|
+
"diffbot-user",
|
|
421
|
+
"doubaobot",
|
|
422
|
+
"duckassistbot",
|
|
190
423
|
"echobot bot",
|
|
191
424
|
"echoboxbot",
|
|
425
|
+
"erniebot",
|
|
426
|
+
"exabot",
|
|
427
|
+
"exasearchbot",
|
|
428
|
+
"exauser",
|
|
429
|
+
"facebookbot",
|
|
192
430
|
"factset_spyderbot",
|
|
431
|
+
"firecrawlagent",
|
|
193
432
|
"friendlycrawler",
|
|
433
|
+
"geisthaus-pagefetcher",
|
|
434
|
+
"gemini-deep-research",
|
|
435
|
+
"google-ads-creatives-assistant",
|
|
436
|
+
"google-agent",
|
|
437
|
+
"google-cloudvertexbot",
|
|
438
|
+
"google-firebase",
|
|
439
|
+
"google-gemini-cli",
|
|
440
|
+
"google-gemininotebook",
|
|
441
|
+
"google-lens",
|
|
442
|
+
"google-notebooklm",
|
|
443
|
+
"googleagent-mariner",
|
|
444
|
+
"googleagent-urlcontext",
|
|
445
|
+
"googleother",
|
|
446
|
+
"googleother-image",
|
|
447
|
+
"googleother-video",
|
|
448
|
+
"gptbot",
|
|
449
|
+
"grove/1.0 rss feed service",
|
|
450
|
+
"henkbot",
|
|
451
|
+
"iaskbot",
|
|
452
|
+
"iaskspider",
|
|
194
453
|
"icc-crawler",
|
|
195
454
|
"imagesiftbot",
|
|
196
455
|
"imagespider",
|
|
197
456
|
"img2dataset",
|
|
198
457
|
"isscyberriskcrawler",
|
|
458
|
+
"jackjillcareersbot",
|
|
459
|
+
"kagi-fetcher",
|
|
199
460
|
"kangaroo bot",
|
|
461
|
+
"keenablebot",
|
|
462
|
+
"kernel searchbot",
|
|
463
|
+
"kimi-agent",
|
|
464
|
+
"kimi-searchbot",
|
|
465
|
+
"kimi-user",
|
|
466
|
+
"kimibot",
|
|
467
|
+
"klaviyoaibot",
|
|
200
468
|
"kunatocrawler",
|
|
201
469
|
"laion-huggingface-processor",
|
|
202
470
|
"laiondownloader",
|
|
471
|
+
"lightpanda",
|
|
472
|
+
"linerbot",
|
|
473
|
+
"linkupbot",
|
|
474
|
+
"manus-user",
|
|
475
|
+
"meta-externalagent",
|
|
476
|
+
"meta-externalfetcher",
|
|
477
|
+
"meta-webindexer",
|
|
478
|
+
"mistralai-index",
|
|
479
|
+
"mistralai-training",
|
|
480
|
+
"mistralai-user",
|
|
481
|
+
"momentic",
|
|
482
|
+
"mozilla-tabstack",
|
|
203
483
|
"mycentralaiscraperbot",
|
|
204
484
|
"nagetbot",
|
|
205
485
|
"netestate imprint crawler",
|
|
486
|
+
"notebooklm",
|
|
487
|
+
"novaact",
|
|
488
|
+
"oai-adsbot",
|
|
489
|
+
"oai-searchbot",
|
|
206
490
|
"omgili",
|
|
207
491
|
"omgilibot",
|
|
492
|
+
"pangubot",
|
|
208
493
|
"panscient",
|
|
209
|
-
"
|
|
210
|
-
"sbintuitionsbot",
|
|
211
|
-
"sidetrade indexer bot",
|
|
212
|
-
"thinkbot",
|
|
213
|
-
"velenpublicwebcrawler",
|
|
214
|
-
"wardbot",
|
|
215
|
-
"oai-searchbot",
|
|
216
|
-
"oai-adsbot",
|
|
217
|
-
"claude-searchbot",
|
|
494
|
+
"perplexity-user",
|
|
218
495
|
"perplexitybot",
|
|
219
|
-
"applebot",
|
|
220
|
-
"amazonbot",
|
|
221
|
-
"amzn-searchbot",
|
|
222
|
-
"amazon-kendra",
|
|
223
|
-
"meta-webindexer",
|
|
224
|
-
"mistralai-index",
|
|
225
|
-
"kimi-searchbot",
|
|
226
|
-
"azureai-searchbot",
|
|
227
496
|
"petalbot",
|
|
228
|
-
"
|
|
229
|
-
"
|
|
230
|
-
"
|
|
231
|
-
"
|
|
232
|
-
"
|
|
497
|
+
"phindbot",
|
|
498
|
+
"poggio-citations",
|
|
499
|
+
"poseidon research crawler",
|
|
500
|
+
"qodercli",
|
|
501
|
+
"qualifiedbot",
|
|
502
|
+
"querit-searchbot",
|
|
503
|
+
"queritbot",
|
|
504
|
+
"quillbot",
|
|
505
|
+
"qwenbot",
|
|
506
|
+
"reflectionbot",
|
|
507
|
+
"ryebot",
|
|
508
|
+
"sbintuitionsbot",
|
|
233
509
|
"seltzsitemapfetcher",
|
|
510
|
+
"semanticscholarbot",
|
|
511
|
+
"semrushbot-ocob",
|
|
512
|
+
"semrushbot-swa",
|
|
513
|
+
"shap-user",
|
|
234
514
|
"shapbot",
|
|
515
|
+
"sidetrade indexer bot",
|
|
235
516
|
"tavilybot",
|
|
236
|
-
"bravebot",
|
|
237
|
-
"cloudflare-autorag",
|
|
238
|
-
"jackjillcareersbot",
|
|
239
|
-
"addsearchbot",
|
|
240
|
-
"aiwebindex",
|
|
241
|
-
"andibot",
|
|
242
|
-
"aranet-searchbot",
|
|
243
|
-
"channel3bot",
|
|
244
|
-
"iaskbot",
|
|
245
|
-
"iaskspider",
|
|
246
|
-
"querit-searchbot",
|
|
247
|
-
"queritbot",
|
|
248
|
-
"terracotta",
|
|
249
517
|
"terra cotta",
|
|
250
|
-
"
|
|
251
|
-
"
|
|
252
|
-
"
|
|
253
|
-
"
|
|
254
|
-
"claude-web",
|
|
255
|
-
"claude-code",
|
|
256
|
-
"perplexity-user",
|
|
257
|
-
"mistralai-user",
|
|
258
|
-
"meta-externalfetcher",
|
|
259
|
-
"google-cloudvertexbot",
|
|
260
|
-
"cloudvertexbot",
|
|
261
|
-
"google-firebase",
|
|
262
|
-
"google-notebooklm",
|
|
263
|
-
"notebooklm",
|
|
264
|
-
"gemini-deep-research",
|
|
265
|
-
"google-agent",
|
|
266
|
-
"googleagent-mariner",
|
|
267
|
-
"googleagent-urlcontext",
|
|
268
|
-
"google-gemini-cli",
|
|
269
|
-
"duckassistbot",
|
|
270
|
-
"cohere-ai",
|
|
271
|
-
"amzn-user",
|
|
272
|
-
"amazonbuyforme",
|
|
273
|
-
"amazon-qbusiness",
|
|
274
|
-
"novaact",
|
|
275
|
-
"bedrockbot",
|
|
276
|
-
"diffbot-user",
|
|
277
|
-
"ai2bot-deepresearcheval",
|
|
278
|
-
"kimi-agent",
|
|
279
|
-
"kimi-user",
|
|
280
|
-
"manus-user",
|
|
281
|
-
"kagi-fetcher",
|
|
282
|
-
"phindbot",
|
|
518
|
+
"terracotta",
|
|
519
|
+
"thinkbot",
|
|
520
|
+
"tiktokspider",
|
|
521
|
+
"timpibot",
|
|
283
522
|
"tongyibot",
|
|
284
|
-
"
|
|
285
|
-
"doubaobot",
|
|
286
|
-
"shap-user",
|
|
287
|
-
"firecrawlagent",
|
|
288
|
-
"apifybot",
|
|
289
|
-
"apifywebsitecontentcrawler",
|
|
290
|
-
"cloudflarebrowserrenderingcrawler",
|
|
291
|
-
"crawlspace",
|
|
292
|
-
"mozilla-tabstack",
|
|
293
|
-
"klaviyoaibot",
|
|
294
|
-
"atlassian-bot",
|
|
295
|
-
"semrushbot-ocob",
|
|
296
|
-
"semrushbot-swa",
|
|
297
|
-
"quillbot",
|
|
298
|
-
"bigsur.ai",
|
|
299
|
-
"geisthaus-pagefetcher",
|
|
300
|
-
"poggio-citations",
|
|
301
|
-
"qualifiedbot",
|
|
523
|
+
"twilioknowledge",
|
|
302
524
|
"twinagent",
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
"
|
|
306
|
-
"
|
|
307
|
-
"
|
|
308
|
-
"
|
|
309
|
-
"
|
|
310
|
-
"
|
|
311
|
-
"
|
|
312
|
-
"wrtnbot"
|
|
525
|
+
"v0bot",
|
|
526
|
+
"velenpublicwebcrawler",
|
|
527
|
+
"wardbot",
|
|
528
|
+
"wrtnbot",
|
|
529
|
+
"yandexadditional",
|
|
530
|
+
"yandexadditionalbot",
|
|
531
|
+
"yiyanbot",
|
|
532
|
+
"youbot",
|
|
533
|
+
"zanistabot"
|
|
313
534
|
]
|
|
314
535
|
};
|
|
315
536
|
|
|
@@ -629,6 +850,27 @@ var ip_ranges_generated_default = {
|
|
|
629
850
|
],
|
|
630
851
|
ipv6: []
|
|
631
852
|
},
|
|
853
|
+
{
|
|
854
|
+
bot: "ClaudeBot",
|
|
855
|
+
url: "https://claude.com/crawling/bots.json",
|
|
856
|
+
ipv4: [
|
|
857
|
+
"34.152.124.140/32",
|
|
858
|
+
"34.153.21.190/32",
|
|
859
|
+
"34.162.27.34/32",
|
|
860
|
+
"34.162.86.102/32",
|
|
861
|
+
"34.162.90.102/32",
|
|
862
|
+
"34.162.111.78/32",
|
|
863
|
+
"34.162.153.71/32",
|
|
864
|
+
"34.162.191.81/32",
|
|
865
|
+
"34.162.230.222/32",
|
|
866
|
+
"34.162.244.71/32",
|
|
867
|
+
"34.186.252.4/32",
|
|
868
|
+
"136.79.143.251/32",
|
|
869
|
+
"136.83.20.215/32",
|
|
870
|
+
"216.73.216.0/22"
|
|
871
|
+
],
|
|
872
|
+
ipv6: []
|
|
873
|
+
},
|
|
632
874
|
{
|
|
633
875
|
bot: "PerplexityBot",
|
|
634
876
|
url: "https://www.perplexity.com/perplexitybot.json",
|
|
@@ -1182,6 +1424,46 @@ var ip_ranges_generated_default = {
|
|
|
1182
1424
|
"191.235.202.48/32"
|
|
1183
1425
|
],
|
|
1184
1426
|
ipv6: []
|
|
1427
|
+
},
|
|
1428
|
+
{
|
|
1429
|
+
bot: "Applebot",
|
|
1430
|
+
url: "https://search.developer.apple.com/applebot.json",
|
|
1431
|
+
ipv4: [
|
|
1432
|
+
"17.166.20.0/24",
|
|
1433
|
+
"17.166.21.0/24",
|
|
1434
|
+
"17.166.22.0/24",
|
|
1435
|
+
"17.166.23.0/24",
|
|
1436
|
+
"17.166.24.0/24",
|
|
1437
|
+
"17.166.25.0/24",
|
|
1438
|
+
"17.166.150.0/24",
|
|
1439
|
+
"17.166.151.0/24",
|
|
1440
|
+
"17.166.152.0/24",
|
|
1441
|
+
"17.166.153.0/24",
|
|
1442
|
+
"17.166.154.0/24",
|
|
1443
|
+
"17.166.155.0/24",
|
|
1444
|
+
"17.166.192.48/28",
|
|
1445
|
+
"17.166.232.0/24",
|
|
1446
|
+
"17.166.233.0/24",
|
|
1447
|
+
"17.166.234.0/24",
|
|
1448
|
+
"17.166.235.0/24",
|
|
1449
|
+
"17.166.236.0/24",
|
|
1450
|
+
"17.166.237.0/24",
|
|
1451
|
+
"17.166.240.48/28",
|
|
1452
|
+
"17.166.248.48/28",
|
|
1453
|
+
"17.241.193.160/27",
|
|
1454
|
+
"17.241.200.160/27",
|
|
1455
|
+
"17.241.208.160/27"
|
|
1456
|
+
],
|
|
1457
|
+
ipv6: []
|
|
1458
|
+
},
|
|
1459
|
+
{
|
|
1460
|
+
bot: "Claude API",
|
|
1461
|
+
url: "https://platform.claude.com/docs/en/api/ip-addresses.md",
|
|
1462
|
+
ipv4: [
|
|
1463
|
+
"153.61.208.0/20",
|
|
1464
|
+
"160.79.104.0/21"
|
|
1465
|
+
],
|
|
1466
|
+
ipv6: []
|
|
1185
1467
|
}
|
|
1186
1468
|
]
|
|
1187
1469
|
};
|
|
@@ -1189,29 +1471,42 @@ var ip_ranges_generated_default = {
|
|
|
1189
1471
|
// src/logic/decide.ts
|
|
1190
1472
|
var MAX_SCANNED_USER_AGENT = 1024;
|
|
1191
1473
|
var PREFIX = 3;
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
const
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
list
|
|
1198
|
-
|
|
1199
|
-
|
|
1474
|
+
function compile(rules) {
|
|
1475
|
+
const tokensByPrefix = /* @__PURE__ */ new Map();
|
|
1476
|
+
for (const token of rules.tokens) {
|
|
1477
|
+
const key = token.slice(0, PREFIX);
|
|
1478
|
+
const list = tokensByPrefix.get(key);
|
|
1479
|
+
if (list) {
|
|
1480
|
+
list.push(token);
|
|
1481
|
+
} else {
|
|
1482
|
+
tokensByPrefix.set(key, [token]);
|
|
1483
|
+
}
|
|
1200
1484
|
}
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1485
|
+
const agentRanges = new BlockList();
|
|
1486
|
+
const addRanges = (cidrs, type) => {
|
|
1487
|
+
for (const cidr of cidrs) {
|
|
1488
|
+
const [address = "", prefix, extra] = cidr.split("/");
|
|
1489
|
+
if (!prefix || extra !== void 0) {
|
|
1490
|
+
throw new Error(`malformed range ${cidr}`);
|
|
1491
|
+
}
|
|
1492
|
+
agentRanges.addSubnet(address, Number(prefix), type);
|
|
1493
|
+
}
|
|
1494
|
+
};
|
|
1495
|
+
for (const source of rules.ipRanges) {
|
|
1496
|
+
addRanges(source.ipv4, "ipv4");
|
|
1497
|
+
addRanges(source.ipv6, "ipv6");
|
|
1207
1498
|
}
|
|
1208
|
-
};
|
|
1209
|
-
for (const source of ip_ranges_generated_default.sources) {
|
|
1210
|
-
addRanges(source.ipv4, "ipv4");
|
|
1211
|
-
addRanges(source.ipv6, "ipv6");
|
|
1499
|
+
return { tokensByPrefix, agentRanges };
|
|
1212
1500
|
}
|
|
1213
|
-
var
|
|
1214
|
-
|
|
1501
|
+
var bundled = compile({ tokens: agents_generated_default.tokens, ipRanges: ip_ranges_generated_default.sources });
|
|
1502
|
+
var remote = null;
|
|
1503
|
+
function applyRules(rules) {
|
|
1504
|
+
remote = compile(rules);
|
|
1505
|
+
}
|
|
1506
|
+
var pick = (useBundled) => useBundled ? bundled : remote ?? bundled;
|
|
1507
|
+
var isAgentIp = (ip, useBundled = false) => pick(useBundled).agentRanges.check(ip, ip.includes(":") ? "ipv6" : "ipv4");
|
|
1508
|
+
function hasAgentToken(userAgent, useBundled = false) {
|
|
1509
|
+
const { tokensByPrefix } = pick(useBundled);
|
|
1215
1510
|
const text = userAgent.slice(0, MAX_SCANNED_USER_AGENT).toLowerCase();
|
|
1216
1511
|
for (let i = 0; i + PREFIX <= text.length; i++) {
|
|
1217
1512
|
const candidates = tokensByPrefix.get(text.slice(i, i + PREFIX));
|
|
@@ -1225,9 +1520,13 @@ function hasAgentToken(userAgent) {
|
|
|
1225
1520
|
}
|
|
1226
1521
|
return false;
|
|
1227
1522
|
}
|
|
1228
|
-
function decide({
|
|
1523
|
+
function decide({
|
|
1524
|
+
headers,
|
|
1525
|
+
ip,
|
|
1526
|
+
useBundled = false
|
|
1527
|
+
}) {
|
|
1229
1528
|
const userAgent = headers["user-agent"];
|
|
1230
|
-
return userAgent !== void 0 && hasAgentToken(userAgent) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip);
|
|
1529
|
+
return userAgent !== void 0 && hasAgentToken(userAgent, useBundled) || headers["signature-agent"] !== void 0 || headers.dpop !== void 0 || headers.authorization?.toLowerCase() === "dpop" || ip !== null && isAgentIp(ip, useBundled);
|
|
1231
1530
|
}
|
|
1232
1531
|
|
|
1233
1532
|
// src/match-key.ts
|
|
@@ -1261,6 +1560,62 @@ async function deriveMatchKey(orgKey, ip) {
|
|
|
1261
1560
|
return toHex(signature);
|
|
1262
1561
|
}
|
|
1263
1562
|
|
|
1563
|
+
// src/refresh.ts
|
|
1564
|
+
var isObject2 = (value) => typeof value === "object" && value !== null;
|
|
1565
|
+
var isStringArray2 = (value) => Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
1566
|
+
function parseRemoteRules(body) {
|
|
1567
|
+
if (!isObject2(body) || body.version !== 1) {
|
|
1568
|
+
return null;
|
|
1569
|
+
}
|
|
1570
|
+
const { tokens, ipRanges } = body;
|
|
1571
|
+
if (!isStringArray2(tokens) || !Array.isArray(ipRanges)) {
|
|
1572
|
+
return null;
|
|
1573
|
+
}
|
|
1574
|
+
const ranges = [];
|
|
1575
|
+
for (const range of ipRanges) {
|
|
1576
|
+
if (!isObject2(range) || !isStringArray2(range.ipv4) || !isStringArray2(range.ipv6)) {
|
|
1577
|
+
return null;
|
|
1578
|
+
}
|
|
1579
|
+
ranges.push({ ipv4: range.ipv4, ipv6: range.ipv6 });
|
|
1580
|
+
}
|
|
1581
|
+
return {
|
|
1582
|
+
tokens: tokens.map((token) => token.toLowerCase()),
|
|
1583
|
+
ipRanges: ranges
|
|
1584
|
+
};
|
|
1585
|
+
}
|
|
1586
|
+
var lastAttemptOk = false;
|
|
1587
|
+
var lastAttemptAt = Number.NEGATIVE_INFINITY;
|
|
1588
|
+
async function fetchRules(apiUrl) {
|
|
1589
|
+
const response = await fetch(`${apiUrl}/api/v2/agent-traffic/rules`, {
|
|
1590
|
+
signal: AbortSignal.timeout(settings.requestTimeoutMs)
|
|
1591
|
+
});
|
|
1592
|
+
if (!response.ok) {
|
|
1593
|
+
return false;
|
|
1594
|
+
}
|
|
1595
|
+
const body = await response.json();
|
|
1596
|
+
const rules = parseRemoteRules(isObject2(body) ? body.data : null);
|
|
1597
|
+
if (!rules) {
|
|
1598
|
+
return false;
|
|
1599
|
+
}
|
|
1600
|
+
applyRules(rules);
|
|
1601
|
+
return true;
|
|
1602
|
+
}
|
|
1603
|
+
function refreshRules(apiUrl) {
|
|
1604
|
+
const now = Date.now();
|
|
1605
|
+
const wait = lastAttemptOk ? settings.rulesRefreshMs : settings.rulesRetryMs;
|
|
1606
|
+
if (now - lastAttemptAt < wait) {
|
|
1607
|
+
return null;
|
|
1608
|
+
}
|
|
1609
|
+
lastAttemptAt = now;
|
|
1610
|
+
lastAttemptOk = false;
|
|
1611
|
+
return fetchRules(apiUrl).then(
|
|
1612
|
+
(ok) => {
|
|
1613
|
+
lastAttemptOk = ok;
|
|
1614
|
+
},
|
|
1615
|
+
() => void 0
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1264
1619
|
// src/wait-until.ts
|
|
1265
1620
|
var REQUEST_CONTEXT = /* @__PURE__ */ Symbol.for("@vercel/request-context");
|
|
1266
1621
|
function vercelRequestContext() {
|
|
@@ -1306,6 +1661,14 @@ function originOf(host) {
|
|
|
1306
1661
|
return null;
|
|
1307
1662
|
}
|
|
1308
1663
|
}
|
|
1664
|
+
function hostnameOf(host) {
|
|
1665
|
+
const origin = originOf(host);
|
|
1666
|
+
return origin === null ? null : new URL(origin).hostname;
|
|
1667
|
+
}
|
|
1668
|
+
function cloudFrontHeader(input, name) {
|
|
1669
|
+
const value = input.headers?.[name]?.[0]?.value;
|
|
1670
|
+
return typeof value === "string" ? value : null;
|
|
1671
|
+
}
|
|
1309
1672
|
function cloudFrontHeaders(input) {
|
|
1310
1673
|
const headers = new Headers();
|
|
1311
1674
|
for (const name of HEADER_CAP) {
|
|
@@ -1343,18 +1706,33 @@ function toRelayRequest(input) {
|
|
|
1343
1706
|
};
|
|
1344
1707
|
}
|
|
1345
1708
|
|
|
1346
|
-
// src/version.ts
|
|
1347
|
-
var RELAY_VERSION = true ? "0.3.0" : "0.0.0-dev";
|
|
1348
|
-
|
|
1349
1709
|
// src/index.ts
|
|
1350
|
-
function isAgent(request) {
|
|
1710
|
+
function isAgent(request, options = {}) {
|
|
1711
|
+
startRulesRefresh(options);
|
|
1351
1712
|
try {
|
|
1352
1713
|
const { headers, ip } = toRelayRequest(request);
|
|
1353
|
-
return decide({
|
|
1714
|
+
return decide({
|
|
1715
|
+
headers: readCappedHeaders(headers),
|
|
1716
|
+
ip,
|
|
1717
|
+
useBundled: options.rules === "bundled"
|
|
1718
|
+
});
|
|
1354
1719
|
} catch {
|
|
1355
1720
|
return false;
|
|
1356
1721
|
}
|
|
1357
1722
|
}
|
|
1723
|
+
function startRulesRefresh(options) {
|
|
1724
|
+
try {
|
|
1725
|
+
if (options.rules === "bundled") {
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
const apiUrl = resolveApiUrl(options);
|
|
1729
|
+
const work = apiUrl ? refreshRules(apiUrl) : null;
|
|
1730
|
+
if (work) {
|
|
1731
|
+
schedule(work, options.waitUntil);
|
|
1732
|
+
}
|
|
1733
|
+
} catch {
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1358
1736
|
function recordAgentVisit(request, options = {}) {
|
|
1359
1737
|
try {
|
|
1360
1738
|
const key = readKey(options.key);
|
|
@@ -1421,6 +1799,216 @@ function recordAgentVisit(request, options = {}) {
|
|
|
1421
1799
|
return Promise.resolve();
|
|
1422
1800
|
}
|
|
1423
1801
|
}
|
|
1802
|
+
var BLOCKED = {
|
|
1803
|
+
status: 403,
|
|
1804
|
+
headers: {
|
|
1805
|
+
"content-type": "text/plain; charset=utf-8",
|
|
1806
|
+
"x-waniwani-blocked": "1",
|
|
1807
|
+
"cache-control": "private, no-store",
|
|
1808
|
+
vary: "user-agent"
|
|
1809
|
+
}
|
|
1810
|
+
};
|
|
1811
|
+
async function serveAgentPage(request, options = {}) {
|
|
1812
|
+
try {
|
|
1813
|
+
return isCloudFrontRequest(request) ? await serveCloudFront(request, options) : await serveFetch(request, options);
|
|
1814
|
+
} catch {
|
|
1815
|
+
return null;
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
async function serveFetch(request, options) {
|
|
1819
|
+
const url = new URL(request.url);
|
|
1820
|
+
const answer = await answerAgentPage(
|
|
1821
|
+
{
|
|
1822
|
+
method: request.method.toUpperCase(),
|
|
1823
|
+
hostname: url.hostname,
|
|
1824
|
+
pathname: url.pathname,
|
|
1825
|
+
userAgent: request.headers.get("user-agent") ?? "",
|
|
1826
|
+
rendering: request.headers.has("x-waniwani-render")
|
|
1827
|
+
},
|
|
1828
|
+
options
|
|
1829
|
+
);
|
|
1830
|
+
if (!answer) {
|
|
1831
|
+
return null;
|
|
1832
|
+
}
|
|
1833
|
+
return new Response(answer.body, {
|
|
1834
|
+
status: answer.status,
|
|
1835
|
+
headers: answer.headers
|
|
1836
|
+
});
|
|
1837
|
+
}
|
|
1838
|
+
async function serveCloudFront(request, options) {
|
|
1839
|
+
const hostname = takeMarkFromUri(request);
|
|
1840
|
+
if (hostname === null) {
|
|
1841
|
+
return null;
|
|
1842
|
+
}
|
|
1843
|
+
const method = request.method.toUpperCase();
|
|
1844
|
+
if (!isPageMethod(method)) {
|
|
1845
|
+
return null;
|
|
1846
|
+
}
|
|
1847
|
+
const match = await findSite(hostname, options);
|
|
1848
|
+
if (!match) {
|
|
1849
|
+
return null;
|
|
1850
|
+
}
|
|
1851
|
+
const answer = await pageAnswer(
|
|
1852
|
+
match.rules,
|
|
1853
|
+
hostname,
|
|
1854
|
+
normalizePath(request.uri),
|
|
1855
|
+
method
|
|
1856
|
+
);
|
|
1857
|
+
if (!answer) {
|
|
1858
|
+
return null;
|
|
1859
|
+
}
|
|
1860
|
+
const result = toCloudFrontResponse(answer);
|
|
1861
|
+
if (result.body !== void 0 && new TextEncoder().encode(result.body).byteLength > settings.cloudFrontMaxBodyBytes) {
|
|
1862
|
+
return null;
|
|
1863
|
+
}
|
|
1864
|
+
return result;
|
|
1865
|
+
}
|
|
1866
|
+
function toCloudFrontResponse(answer) {
|
|
1867
|
+
const headers = {};
|
|
1868
|
+
for (const [name, value] of Object.entries(answer.headers)) {
|
|
1869
|
+
headers[name] = [{ key: headerKey(name), value }];
|
|
1870
|
+
}
|
|
1871
|
+
const result = {
|
|
1872
|
+
status: String(answer.status),
|
|
1873
|
+
statusDescription: answer.status === 200 ? "OK" : "Forbidden",
|
|
1874
|
+
headers
|
|
1875
|
+
};
|
|
1876
|
+
if (answer.body !== null) {
|
|
1877
|
+
result.body = typeof answer.body === "string" ? answer.body : new TextDecoder().decode(answer.body);
|
|
1878
|
+
}
|
|
1879
|
+
return result;
|
|
1880
|
+
}
|
|
1881
|
+
function headerKey(name) {
|
|
1882
|
+
return name.split("-").map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join("-");
|
|
1883
|
+
}
|
|
1884
|
+
var isPageMethod = (method) => method === "GET" || method === "HEAD";
|
|
1885
|
+
async function findSite(hostname, options) {
|
|
1886
|
+
const key = readKey(options.key);
|
|
1887
|
+
if (!key) {
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
const apiUrl = resolveApiUrl(options);
|
|
1891
|
+
if (!apiUrl) {
|
|
1892
|
+
return null;
|
|
1893
|
+
}
|
|
1894
|
+
const rules = await loadSiteRules(
|
|
1895
|
+
apiUrl,
|
|
1896
|
+
key.ingestToken,
|
|
1897
|
+
(work) => schedule(work, void 0)
|
|
1898
|
+
);
|
|
1899
|
+
if (!rules) {
|
|
1900
|
+
return null;
|
|
1901
|
+
}
|
|
1902
|
+
const site = rules.sites.find((entry) => entry.hostname === hostname);
|
|
1903
|
+
return site ? { rules, site } : null;
|
|
1904
|
+
}
|
|
1905
|
+
function blockedAnswer(site, userAgent, method) {
|
|
1906
|
+
return site.blockedUserAgentTokens.some((token) => userAgent.includes(token)) ? { ...BLOCKED, body: method === "HEAD" ? null : "Forbidden" } : null;
|
|
1907
|
+
}
|
|
1908
|
+
async function answerAgentPage(request, options) {
|
|
1909
|
+
const { method, hostname } = request;
|
|
1910
|
+
if (!isPageMethod(method)) {
|
|
1911
|
+
return null;
|
|
1912
|
+
}
|
|
1913
|
+
const match = await findSite(hostname, options);
|
|
1914
|
+
if (!match) {
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
const { rules, site } = match;
|
|
1918
|
+
const userAgent = request.userAgent.toLowerCase();
|
|
1919
|
+
const blocked = blockedAnswer(site, userAgent, method);
|
|
1920
|
+
if (blocked) {
|
|
1921
|
+
return blocked;
|
|
1922
|
+
}
|
|
1923
|
+
if (request.rendering || !site.agentPages.enabled) {
|
|
1924
|
+
return null;
|
|
1925
|
+
}
|
|
1926
|
+
const path = normalizePath(request.pathname);
|
|
1927
|
+
if (!matchesPath(path, site.agentPages.paths)) {
|
|
1928
|
+
return null;
|
|
1929
|
+
}
|
|
1930
|
+
if (!rules.userAgentTokens.some((token) => userAgent.includes(token))) {
|
|
1931
|
+
return null;
|
|
1932
|
+
}
|
|
1933
|
+
return pageAnswer(rules, hostname, path, method);
|
|
1934
|
+
}
|
|
1935
|
+
async function pageAnswer(rules, hostname, path, method) {
|
|
1936
|
+
const page = await fetchAgentPage(rules, hostname, path);
|
|
1937
|
+
if (!page) {
|
|
1938
|
+
return null;
|
|
1939
|
+
}
|
|
1940
|
+
return {
|
|
1941
|
+
status: 200,
|
|
1942
|
+
headers: {
|
|
1943
|
+
"content-type": page.contentType,
|
|
1944
|
+
"x-waniwani-agent-page": "hit",
|
|
1945
|
+
"cache-control": "private, no-store",
|
|
1946
|
+
vary: "user-agent, accept"
|
|
1947
|
+
},
|
|
1948
|
+
body: method === "HEAD" ? null : page.body
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
var AGENT_URI_PREFIX = "/__waniwani-agent/";
|
|
1952
|
+
function splitAgentUri(uri) {
|
|
1953
|
+
if (!uri.startsWith(AGENT_URI_PREFIX)) {
|
|
1954
|
+
return null;
|
|
1955
|
+
}
|
|
1956
|
+
const rest = uri.slice(AGENT_URI_PREFIX.length);
|
|
1957
|
+
const slash = rest.indexOf("/");
|
|
1958
|
+
return slash === -1 ? { encodedHost: rest, originalUri: "/" } : { encodedHost: rest.slice(0, slash), originalUri: rest.slice(slash) };
|
|
1959
|
+
}
|
|
1960
|
+
function stripForgedMark(request) {
|
|
1961
|
+
let forged = splitAgentUri(request.uri);
|
|
1962
|
+
while (forged) {
|
|
1963
|
+
request.uri = forged.originalUri;
|
|
1964
|
+
forged = splitAgentUri(request.uri);
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
function takeMarkFromUri(request) {
|
|
1968
|
+
const parts = splitAgentUri(request.uri);
|
|
1969
|
+
if (!parts) {
|
|
1970
|
+
return null;
|
|
1971
|
+
}
|
|
1972
|
+
request.uri = parts.originalUri;
|
|
1973
|
+
try {
|
|
1974
|
+
return hostnameOf(decodeURIComponent(parts.encodedHost));
|
|
1975
|
+
} catch {
|
|
1976
|
+
return null;
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
async function handleViewerRequest(request, options = {}) {
|
|
1980
|
+
try {
|
|
1981
|
+
stripForgedMark(request);
|
|
1982
|
+
const method = request.method.toUpperCase();
|
|
1983
|
+
if (!isPageMethod(method) || !isAgent(request, options)) {
|
|
1984
|
+
return null;
|
|
1985
|
+
}
|
|
1986
|
+
const hostname = hostnameOf(cloudFrontHeader(request, "host") ?? void 0);
|
|
1987
|
+
if (hostname === null) {
|
|
1988
|
+
return null;
|
|
1989
|
+
}
|
|
1990
|
+
const match = await findSite(hostname, options);
|
|
1991
|
+
if (!match) {
|
|
1992
|
+
return null;
|
|
1993
|
+
}
|
|
1994
|
+
const { rules, site } = match;
|
|
1995
|
+
const userAgent = (cloudFrontHeader(request, "user-agent") ?? "").toLowerCase();
|
|
1996
|
+
const blocked = blockedAnswer(site, userAgent, method);
|
|
1997
|
+
if (blocked) {
|
|
1998
|
+
return toCloudFrontResponse(blocked);
|
|
1999
|
+
}
|
|
2000
|
+
if (cloudFrontHeader(request, "x-waniwani-render") !== null || !site.agentPages.enabled || !matchesPath(normalizePath(request.uri), site.agentPages.paths) || !rules.userAgentTokens.some((token) => userAgent.includes(token))) {
|
|
2001
|
+
return null;
|
|
2002
|
+
}
|
|
2003
|
+
const uri = `${AGENT_URI_PREFIX}${encodeURIComponent(hostname)}${request.uri}`;
|
|
2004
|
+
if (new TextEncoder().encode(uri).byteLength <= settings.cloudFrontMaxUriBytes) {
|
|
2005
|
+
request.uri = uri;
|
|
2006
|
+
}
|
|
2007
|
+
return null;
|
|
2008
|
+
} catch {
|
|
2009
|
+
return null;
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
1424
2012
|
var warnedRegion = false;
|
|
1425
2013
|
function resolveApiUrl(options) {
|
|
1426
2014
|
if (options.apiUrl !== void 0) {
|
|
@@ -1457,7 +2045,9 @@ function schedule(work, waitUntil) {
|
|
|
1457
2045
|
return settled;
|
|
1458
2046
|
}
|
|
1459
2047
|
export {
|
|
2048
|
+
handleViewerRequest,
|
|
1460
2049
|
isAgent,
|
|
1461
|
-
recordAgentVisit
|
|
2050
|
+
recordAgentVisit,
|
|
2051
|
+
serveAgentPage
|
|
1462
2052
|
};
|
|
1463
2053
|
//# sourceMappingURL=index.js.map
|