@osqd/bothandlerjs 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +227 -1
- package/README.md +18 -10
- package/dist/adapters/fastify.d.ts +10 -0
- package/dist/adapters/index.cjs +38 -10
- package/dist/adapters/index.cjs.map +1 -1
- package/dist/adapters/index.js +38 -10
- package/dist/adapters/index.js.map +1 -1
- package/dist/challenge/index.d.ts +40 -0
- package/dist/cli.cjs +2256 -103
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +2256 -103
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +55 -0
- package/dist/core.d.ts +48 -21
- package/dist/corpus/index.cjs +365 -7
- package/dist/corpus/index.cjs.map +1 -1
- package/dist/corpus/index.js +365 -7
- package/dist/corpus/index.js.map +1 -1
- package/dist/corpus/schema.d.ts +33 -0
- package/dist/crawler-ranges.d.ts +31 -0
- package/dist/dashboard/client/actions.d.ts +1 -1
- package/dist/dashboard/client/app.d.ts +9 -2
- package/dist/dashboard/client/boot.d.ts +32 -3
- package/dist/dashboard/client/query.d.ts +72 -12
- package/dist/dashboard/client/registry.d.ts +25 -0
- package/dist/dashboard/client/saved.d.ts +29 -0
- package/dist/dashboard/client/store.d.ts +16 -2
- package/dist/dashboard/client/types.d.ts +2 -0
- package/dist/dashboard/client.generated.d.ts +1 -1
- package/dist/dashboard/types.d.ts +15 -0
- package/dist/detectors/blended-identity.d.ts +34 -0
- package/dist/detectors/challenge-integrity.d.ts +26 -0
- package/dist/detectors/challenge-reaction.d.ts +39 -0
- package/dist/detectors/clearance.d.ts +1 -23
- package/dist/detectors/id-enumeration.d.ts +31 -0
- package/dist/detectors/index.d.ts +24 -1
- package/dist/detectors/known-bots.d.ts +11 -0
- package/dist/detectors/marker.d.ts +106 -0
- package/dist/detectors/parameter-sweep.d.ts +39 -0
- package/dist/detectors/probe-signature.d.ts +27 -0
- package/dist/detectors/probe-volume.d.ts +26 -0
- package/dist/detectors/site-baseline.d.ts +135 -0
- package/dist/detectors/target-integrity.d.ts +16 -0
- package/dist/detectors/transport-coherence.d.ts +31 -0
- package/dist/detectors/trap.d.ts +10 -3
- package/dist/detectors/types.d.ts +17 -0
- package/dist/element/index.cjs +730 -80
- package/dist/element/index.cjs.map +1 -1
- package/dist/element/index.js +730 -80
- package/dist/element/index.js.map +1 -1
- package/dist/index.cjs +2044 -123
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.js +2025 -123
- package/dist/index.js.map +1 -1
- package/dist/internal/async.d.ts +0 -3
- package/dist/internal/ip.d.ts +18 -0
- package/dist/internal/text.d.ts +28 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/probe/index.d.ts +153 -0
- package/dist/probe/marker.d.ts +119 -0
- package/dist/site/index.d.ts +122 -0
- package/dist/state.d.ts +205 -0
- package/dist/stores/redis.d.ts +24 -1
- package/dist/types.d.ts +106 -0
- package/docs/course/05-detectors.md +9 -4
- package/docs/course/06-identity.md +1 -1
- package/docs/course/16-proving-it.md +15 -9
- package/docs/course/index.md +1 -1
- package/docs/design/decisions.md +1 -1
- package/docs/detection/correlation.md +284 -0
- package/docs/detection/detectors.md +259 -1
- package/docs/detection/index.md +2 -1
- package/docs/detection/shadow-mode.md +147 -0
- package/docs/detection/signatures.md +10 -2
- package/docs/index.md +3 -2
- package/docs/integration/client-ip.md +16 -0
- package/docs/operations/dashboard.md +40 -1
- package/docs/operations/filters.md +143 -0
- package/docs/operations/index.md +1 -0
- package/docs/operations/metrics.md +18 -0
- package/docs/policy/presets.md +1 -1
- package/docs/start/choosing-a-policy.md +1 -1
- package/docs/start/first-integration.md +1 -1
- package/docs/start/installation.md +2 -2
- package/docs/testing/cli.md +7 -1
- package/docs/testing/corpus.md +12 -8
- package/docs/testing/index.md +1 -1
- package/docs/testing/try-it.md +1 -1
- package/package.json +4 -1
package/dist/corpus/index.js
CHANGED
|
@@ -310,18 +310,21 @@ function createFacts(input) {
|
|
|
310
310
|
const rawPath = queryStart === -1 ? url : url.slice(0, queryStart);
|
|
311
311
|
const headers = /* @__PURE__ */ Object.create(null);
|
|
312
312
|
for (const [name, value] of Object.entries(input.headers)) {
|
|
313
|
-
const
|
|
314
|
-
|
|
313
|
+
const lower = name.toLowerCase();
|
|
314
|
+
const joined = lower === "cookie" && Array.isArray(value) ? value.join("; ") : joinHeaderValue(value);
|
|
315
|
+
if (joined !== void 0) headers[lower] = joined;
|
|
315
316
|
}
|
|
317
|
+
const normalized = normalizePath(rawPath);
|
|
316
318
|
const facts = {
|
|
317
319
|
method: (input.method ?? "GET").toUpperCase(),
|
|
318
|
-
path:
|
|
320
|
+
path: normalized,
|
|
319
321
|
query: parseQuery(queryStart === -1 ? "" : url.slice(queryStart + 1)),
|
|
320
322
|
headers,
|
|
321
323
|
headerOrder: extractOrder(input.rawHeaders, headers),
|
|
322
324
|
ip: normalizeIp(input.ip) ?? input.ip,
|
|
323
325
|
timestamp: input.timestamp ?? Date.now()
|
|
324
326
|
};
|
|
327
|
+
if (rawPath !== normalized) facts.rawPath = rawPath.length > MAX_RAW_PATH ? rawPath.slice(0, MAX_RAW_PATH) : rawPath;
|
|
325
328
|
const cookieHeader = headers["cookie"];
|
|
326
329
|
if (cookieHeader !== void 0) facts.cookies = parseCookies(cookieHeader);
|
|
327
330
|
if (input.protocol !== void 0) facts.protocol = input.protocol;
|
|
@@ -354,12 +357,23 @@ function parseQuery(search) {
|
|
|
354
357
|
const query = /* @__PURE__ */ Object.create(null);
|
|
355
358
|
if (search.length === 0) return query;
|
|
356
359
|
let count = 0;
|
|
357
|
-
for (const [key, value] of new URLSearchParams(search)) {
|
|
360
|
+
for (const [key, value] of new URLSearchParams(boundedSearch(search))) {
|
|
358
361
|
if (count++ >= MAX_QUERY_PARAMS) break;
|
|
359
362
|
query[key] = value.length > 1024 ? value.slice(0, 1024) : value;
|
|
360
363
|
}
|
|
361
364
|
return query;
|
|
362
365
|
}
|
|
366
|
+
function boundedSearch(search) {
|
|
367
|
+
let seen = 0;
|
|
368
|
+
let at = search.charCodeAt(0) === 63 ? 1 : 0;
|
|
369
|
+
while (at < search.length) {
|
|
370
|
+
let end = search.indexOf("&", at);
|
|
371
|
+
if (end === -1) end = search.length;
|
|
372
|
+
if (end !== at && ++seen > MAX_QUERY_PARAMS) return search.slice(0, at - 1);
|
|
373
|
+
at = end + 1;
|
|
374
|
+
}
|
|
375
|
+
return search;
|
|
376
|
+
}
|
|
363
377
|
function extractOrder(rawHeaders, headers) {
|
|
364
378
|
if (!rawHeaders || rawHeaders.length === 0) return EMPTY_ORDER;
|
|
365
379
|
let isNodeStyle = rawHeaders.length % 2 === 0;
|
|
@@ -395,13 +409,14 @@ function isHeaderName(value) {
|
|
|
395
409
|
}
|
|
396
410
|
return true;
|
|
397
411
|
}
|
|
398
|
-
var MAX_URL_LENGTH, MAX_QUERY_PARAMS, MAX_ORDERED_HEADERS, EMPTY_ORDER;
|
|
412
|
+
var MAX_URL_LENGTH, MAX_RAW_PATH, MAX_QUERY_PARAMS, MAX_ORDERED_HEADERS, EMPTY_ORDER;
|
|
399
413
|
var init_facts = __esm({
|
|
400
414
|
"src/facts.ts"() {
|
|
401
415
|
"use strict";
|
|
402
416
|
init_http();
|
|
403
417
|
init_ip();
|
|
404
418
|
MAX_URL_LENGTH = 8192;
|
|
419
|
+
MAX_RAW_PATH = 512;
|
|
405
420
|
MAX_QUERY_PARAMS = 64;
|
|
406
421
|
MAX_ORDERED_HEADERS = 64;
|
|
407
422
|
EMPTY_ORDER = Object.freeze([]);
|
|
@@ -1144,7 +1159,248 @@ var AI_CRAWLER_CASES = [
|
|
|
1144
1159
|
|
|
1145
1160
|
// src/corpus/adversarial.ts
|
|
1146
1161
|
var CHROME_UA = userAgentOf("chromeWindows");
|
|
1162
|
+
var CURL_UA = "curl/8.4.0";
|
|
1147
1163
|
var ADVERSARIAL_CASES = [
|
|
1164
|
+
bot({
|
|
1165
|
+
id: "two-scanners-one-address",
|
|
1166
|
+
title: "One address arriving as two different security tools",
|
|
1167
|
+
audience: "hostile",
|
|
1168
|
+
category: "scanning",
|
|
1169
|
+
provenance: "The shape of an actual scan: an operator runs more than one tool against a target, and both announce themselves honestly. Each request on its own is a declared bot; the pair is a scan, and that reading does not exist inside either request.",
|
|
1170
|
+
requests: [
|
|
1171
|
+
{ headers: [["Host", "shop.example"], ["User-Agent", "sqlmap/1.7.2#stable (http://sqlmap.org)"], ["Accept", "*/*"]], ip: "198.51.100.66", atMs: 0 },
|
|
1172
|
+
{ headers: [["Host", "shop.example"], ["User-Agent", "Mozilla/5.00 (Nikto/2.5.0) (Evasions:None) (Test:Port Check)"], ["Accept", "*/*"]], ip: "198.51.100.66", atMs: 1e3 }
|
|
1173
|
+
],
|
|
1174
|
+
expect: { verdict: "confirmed-bot", certain: true, detectors: ["blended-identity"] },
|
|
1175
|
+
notes: "Holds under the default address-based actor key, which is what separates it from `identity-rotation`. A NAT gateway presents a hundred browsers \u2014 that is exactly why counting User-Agents there is useless \u2014 and it does not present sqlmap and nikto."
|
|
1176
|
+
}),
|
|
1177
|
+
bot({
|
|
1178
|
+
id: "two-crawler-claims-one-address",
|
|
1179
|
+
title: "One address claiming to be both Googlebot and Bingbot",
|
|
1180
|
+
audience: "hostile",
|
|
1181
|
+
category: "impersonation",
|
|
1182
|
+
provenance: "At most one of these can be true of an address: each operator publishes a proof tied to addresses it controls. The contradiction is visible from the claims alone, with no lookup \u2014 which matters when DNS is unreachable and neither claim can be refuted on its own.",
|
|
1183
|
+
requests: [
|
|
1184
|
+
{ headers: [["Host", "shop.example"], ["User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"], ["Accept", "*/*"]], ip: "198.51.100.67", atMs: 0 },
|
|
1185
|
+
{ headers: [["Host", "shop.example"], ["User-Agent", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"], ["Accept", "*/*"]], ip: "198.51.100.67", atMs: 1e3 }
|
|
1186
|
+
],
|
|
1187
|
+
expect: { detectors: ["blended-identity"] },
|
|
1188
|
+
notes: "Held at `strong` rather than `certain`. Trusting the wrong forwarded header collapses every client onto one address, and then two genuinely different crawlers produce this exact set \u2014 so it may contribute to a denial and may not be the whole of one."
|
|
1189
|
+
}),
|
|
1190
|
+
bot({
|
|
1191
|
+
id: "id-harvest-contiguous",
|
|
1192
|
+
title: "Every profile id in order, with a copied browser header set",
|
|
1193
|
+
audience: "hostile",
|
|
1194
|
+
category: "scraping",
|
|
1195
|
+
provenance: "Harvesting by identifier rather than by link: the shape of an IDOR sweep and of profile collection. Distinct-path breadth reads it as somebody who visited a lot of pages, which is also what it reads when a person works through a documentation site.",
|
|
1196
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.65" }, 40, 800, (index) => `/user/${index + 1}`),
|
|
1197
|
+
expect: {
|
|
1198
|
+
// One `moderate` signal against a flawless header set, like the others here. What
|
|
1199
|
+
// changed is that the walk is now *visible* — before this detector it was scored
|
|
1200
|
+
// identically to a hundred and twenty scattered ids and to ordinary article paths.
|
|
1201
|
+
verdict: "unknown",
|
|
1202
|
+
detectors: ["id-enumeration"]
|
|
1203
|
+
},
|
|
1204
|
+
notes: "What separates this from reading is not which ids were asked for but that they cover a range: people arrive at ids through links, and links do not densely enumerate an integer interval. Held at `moderate` because products in one category often carry consecutive ids, so somebody browsing a catalogue makes a smaller version of this shape."
|
|
1205
|
+
}),
|
|
1206
|
+
bot({
|
|
1207
|
+
id: "wordlist-scan-mostly-misses",
|
|
1208
|
+
title: "A wordlist walked with a copied browser header set, almost all of it missing",
|
|
1209
|
+
audience: "hostile",
|
|
1210
|
+
category: "scanning",
|
|
1211
|
+
provenance: "The oldest tell there is, and the one this library could not see: it decides before the response exists, which is what lets it shape the response and also what hides the status from it. A person browsing does not generate thirty misses in a row; a wordlist does almost nothing else.",
|
|
1212
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.64", status: 404 }, 30, 700, (index) => `/${["admin", "backup", "old", "test", "config", "db"][index % 6]}-${index}`),
|
|
1213
|
+
expect: {
|
|
1214
|
+
// One `moderate` signal against an otherwise flawless header set does not cross the
|
|
1215
|
+
// line, and it should not: a site that has just moved its URLs produces the same
|
|
1216
|
+
// shape from ordinary readers. Raising the ceiling so this case reads better would
|
|
1217
|
+
// be tuning the detector to the test rather than to the traffic.
|
|
1218
|
+
verdict: "unknown",
|
|
1219
|
+
detectors: ["probe-volume"]
|
|
1220
|
+
},
|
|
1221
|
+
notes: "Only counts 404 and 410. A 403 is usually this library's own doing, and counting it would let a rule that challenges an actor manufacture the evidence for having challenged it; a 500 is the site's problem and says nothing about the client. Capped at `moderate` because a site that has just moved its URLs produces this from perfectly ordinary readers."
|
|
1222
|
+
}),
|
|
1223
|
+
bot({
|
|
1224
|
+
id: "browser-claim-over-http-1-0",
|
|
1225
|
+
title: "A perfect Chrome header set, arriving over HTTP/1.0",
|
|
1226
|
+
audience: "hostile",
|
|
1227
|
+
category: "impersonation",
|
|
1228
|
+
provenance: "Most tooling lets you set headers and does not let you choose an HTTP version, so the transport is the half a copied header set does not cover. No shipping browser has offered HTTP/1.0 to a server in well over a decade.",
|
|
1229
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.62", httpVersion: "1.0" }, 30, 900, (index) => `/${["news", "about", "blog", "help", "terms"][index % 5]}`),
|
|
1230
|
+
expect: {
|
|
1231
|
+
// Contributes rather than concludes. On its own, against an otherwise flawless
|
|
1232
|
+
// header set, one `moderate` signal does not reach the threshold — and it should
|
|
1233
|
+
// not, because an intermediary can cause this. Beside anything sharper it does.
|
|
1234
|
+
verdict: "unknown",
|
|
1235
|
+
detectors: ["transport-coherence"]
|
|
1236
|
+
},
|
|
1237
|
+
notes: "Capped at `moderate` because it is not always the client's doing: a few older load balancers speak HTTP/1.0 to the origin, and behind one of those every request looks like this. That is what `transportCoherenceDetector({ legacyHttp: false })` is for, and why this may never deny anybody on its own."
|
|
1238
|
+
}),
|
|
1239
|
+
// ---------------------------------------------------------------------------
|
|
1240
|
+
// The optional sources. None of these can be detected without the operator
|
|
1241
|
+
// switching something on — a marker cookie, or a site-wide baseline — so each
|
|
1242
|
+
// exists to hold that feature to the same standard as everything shipped by
|
|
1243
|
+
// default. See `docs/detection/correlation.md`.
|
|
1244
|
+
// ---------------------------------------------------------------------------
|
|
1245
|
+
bot({
|
|
1246
|
+
id: "marker-held-while-identity-changes",
|
|
1247
|
+
title: "One client presenting a marker it was issued as Chrome, then as curl",
|
|
1248
|
+
audience: "unwanted-bot",
|
|
1249
|
+
category: "evasion",
|
|
1250
|
+
provenance: "Identity rotation, which is invisible without a marker. Correlating by address cannot tell this apart from two people sharing an office connection, so the library declined to guess. A signed cookie removes the ambiguity: both requests carried an HMAC only this server can produce.",
|
|
1251
|
+
requires: ["marker-probe"],
|
|
1252
|
+
requests: [
|
|
1253
|
+
...repeat({ ...browser("chromeWindows"), ip: "198.51.100.81", headers: [...browser("chromeWindows").headers, ["Cookie", "sid=held"]] }, 4, 1500, (i) => `/products/${i}`),
|
|
1254
|
+
...repeat({ ...plain(CURL_UA), ip: "198.51.100.81", headers: [...plain(CURL_UA).headers, ["Cookie", "sid=held"]] }, 4, 1500, (i) => `/products/${i + 4}`).map((request) => ({ ...request, atMs: (request.atMs ?? 0) + 6e3 }))
|
|
1255
|
+
],
|
|
1256
|
+
expect: { verdict: "confirmed-bot", detectors: ["identity-drift"] },
|
|
1257
|
+
notes: "The browser family carries the weight and the platform does not, because a phone with `Request desktop site` changes its platform and is a person. Software does not change what it is."
|
|
1258
|
+
}),
|
|
1259
|
+
bot({
|
|
1260
|
+
id: "marker-never-stored-though-cookies-sent",
|
|
1261
|
+
title: "A client replaying a captured session cookie and storing nothing new",
|
|
1262
|
+
audience: "unwanted-bot",
|
|
1263
|
+
category: "scraping",
|
|
1264
|
+
provenance: "A scraper handed a session header to copy. It sends the one cookie it was configured with on every request and never stores anything the server sets, which a browser with a jar does not do.",
|
|
1265
|
+
requires: ["marker-probe"],
|
|
1266
|
+
keepsCookies: false,
|
|
1267
|
+
requests: repeat(
|
|
1268
|
+
{ ...browser("chromeWindows"), ip: "198.51.100.82", headers: [...browser("chromeWindows").headers, ["Cookie", "sid=captured-elsewhere"]] },
|
|
1269
|
+
9,
|
|
1270
|
+
1200,
|
|
1271
|
+
(i) => `/products/${i}`
|
|
1272
|
+
),
|
|
1273
|
+
expect: { verdict: "unknown", detectors: ["marker-persistence"] },
|
|
1274
|
+
notes: "Deliberately narrower than `session-integrity`, which already reports a client sending no cookie at all. Overlapping them double-counted one observation and the population it landed on was people who block cookies."
|
|
1275
|
+
}),
|
|
1276
|
+
bot({
|
|
1277
|
+
id: "marker-edited-by-its-holder",
|
|
1278
|
+
title: "A client that edited the signed cookie it was given",
|
|
1279
|
+
audience: "unwanted-bot",
|
|
1280
|
+
category: "evasion",
|
|
1281
|
+
provenance: "Browsers do not edit their own cookies. A marker failing its HMAC was altered by whoever held it, and the only reason to alter an opaque signed value is to see what the server does with a different one.",
|
|
1282
|
+
requires: ["marker-probe"],
|
|
1283
|
+
keepsCookies: false,
|
|
1284
|
+
requests: repeat(
|
|
1285
|
+
{ ...browser("chromeWindows"), ip: "198.51.100.83", headers: [...browser("chromeWindows").headers, ["Cookie", "__bh_m=eyJ2IjoxfQ.not-a-signature-this-server-made"]] },
|
|
1286
|
+
4,
|
|
1287
|
+
1500,
|
|
1288
|
+
(i) => `/account/${i}`
|
|
1289
|
+
),
|
|
1290
|
+
expect: { verdict: "unknown", detectors: ["marker-integrity"] },
|
|
1291
|
+
notes: "Stops at `strong` rather than `certain` because a middlebox or a broken cookie jar can mangle a value in transit. That is rare, it is not the client's fault, and it should cost a challenge rather than a door."
|
|
1292
|
+
}),
|
|
1293
|
+
bot({
|
|
1294
|
+
id: "marker-carried-across-a-proxy-pool",
|
|
1295
|
+
title: "One marker presented from twenty different networks",
|
|
1296
|
+
audience: "unwanted-bot",
|
|
1297
|
+
category: "scraping",
|
|
1298
|
+
provenance: "A scraper on a rotating proxy pool that keeps its cookie jar, which most of them do because discarding it breaks the sites they are taking. The marker comes back only from the client that received it, so this is one client across twenty networks.",
|
|
1299
|
+
requires: ["marker-probe"],
|
|
1300
|
+
requests: Array.from({ length: 20 }, (_, index) => ({
|
|
1301
|
+
...browser("chromeWindows"),
|
|
1302
|
+
headers: [...browser("chromeWindows").headers, ["Cookie", "sid=pooled"]],
|
|
1303
|
+
ip: `198.51.${140 + index}.9`,
|
|
1304
|
+
path: `/catalogue/${index}`,
|
|
1305
|
+
atMs: index * 2500
|
|
1306
|
+
})),
|
|
1307
|
+
expect: { verdict: "unknown", detectors: ["marker-fanout"] },
|
|
1308
|
+
notes: "Capped at `moderate` and offered no higher: a phone on a carrier using CGNAT can be renumbered across a great many /24s in the twelve hours a marker lives, and so can anyone whose employer egresses through a rotating pool."
|
|
1309
|
+
}),
|
|
1310
|
+
bot({
|
|
1311
|
+
id: "range-walked-across-many-clients",
|
|
1312
|
+
title: "An id range divided between ten clients so none of them walks enough to notice",
|
|
1313
|
+
audience: "unwanted-bot",
|
|
1314
|
+
category: "scraping",
|
|
1315
|
+
provenance: "The threat every per-actor threshold misses by construction. Split a range across enough addresses and each one is unremarkable, `id-enumeration` fires for nobody, and the range is still walked end to end. It is only visible in the union.",
|
|
1316
|
+
requires: ["site-baseline"],
|
|
1317
|
+
requests: Array.from({ length: 200 }, (_, index) => ({
|
|
1318
|
+
...browser("chromeWindows"),
|
|
1319
|
+
ip: `198.51.${170 + index % 10}.5`,
|
|
1320
|
+
path: `/user/${index + 1}`,
|
|
1321
|
+
atMs: index * 900
|
|
1322
|
+
})),
|
|
1323
|
+
expect: { verdict: "unknown", detectors: ["distributed-walk"] },
|
|
1324
|
+
notes: "Coverage and the revisit ratio must both agree. Many clients on numbered pages is what a catalogue is; what a catalogue also has, and an enumeration does not, is people returning to the same popular items."
|
|
1325
|
+
}),
|
|
1326
|
+
bot({
|
|
1327
|
+
id: "fresh-path-wanted-by-everybody",
|
|
1328
|
+
title: "A path this site never served, requested at once by twenty unrelated clients",
|
|
1329
|
+
audience: "unwanted-bot",
|
|
1330
|
+
category: "recon",
|
|
1331
|
+
provenance: "What a freshly disclosed vulnerability looks like from inside a site: a URL nobody had ever requested is requested by hundreds of unrelated clients within the hour, each making a single request and moving on.",
|
|
1332
|
+
requires: ["site-baseline"],
|
|
1333
|
+
requests: Array.from({ length: 20 }, (_, index) => ({
|
|
1334
|
+
...plain(CURL_UA),
|
|
1335
|
+
ip: `198.51.${190 + index}.11`,
|
|
1336
|
+
path: "/vendor/proprietary-thing/rce.php",
|
|
1337
|
+
status: 404,
|
|
1338
|
+
atMs: index * 3e3
|
|
1339
|
+
})),
|
|
1340
|
+
expect: { verdict: "unknown", detectors: ["path-campaign"] },
|
|
1341
|
+
notes: "The miss rate is required rather than optional. Many clients arriving at once on a brand-new URL is also exactly what a successful launch looks like; what separates them is whether the site had anything to serve."
|
|
1342
|
+
}),
|
|
1343
|
+
bot({
|
|
1344
|
+
id: "missing-far-more-than-this-site-does",
|
|
1345
|
+
title: 'A client answered "not found" far more often than the site answers it at all',
|
|
1346
|
+
audience: "unwanted-bot",
|
|
1347
|
+
category: "recon",
|
|
1348
|
+
provenance: "A fixed miss threshold is wrong on both kinds of site: on one mid-migration it reports everybody, and on a tidy one it stays silent while a client misses a third of the time. The site's own rate is the only honest comparison.",
|
|
1349
|
+
requires: ["site-baseline"],
|
|
1350
|
+
requests: repeat({ ...plain(CURL_UA), ip: "198.51.210.12", status: 404 }, 26, 1100, (i) => `/backup-${i}.sql`),
|
|
1351
|
+
expect: { verdict: "unknown", detectors: ["miss-baseline"] },
|
|
1352
|
+
notes: "Shares the `misses` family with `probe-volume`, which reads the same misses against a fixed threshold. One cause, so the stronger reading stands rather than the two summing."
|
|
1353
|
+
}),
|
|
1354
|
+
bot({
|
|
1355
|
+
id: "solution-farm-replaying-answers",
|
|
1356
|
+
title: "A client answering challenges with solutions that have already been spent",
|
|
1357
|
+
audience: "unwanted-bot",
|
|
1358
|
+
category: "evasion",
|
|
1359
|
+
provenance: "What a solved-challenge farm looks like from the server. A challenge nonce is random, single-use and signed, so a second valid solution for one is the same answer sent twice or one answer handed around \u2014 neither of which a browser does. One replay is a retried POST on a flaky connection, which is why the threshold is not one.",
|
|
1360
|
+
challengeHistory: { replayedSolutions: 4, implausibleSolves: 2 },
|
|
1361
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.71" }, 6, 1500, () => "/account"),
|
|
1362
|
+
expect: {
|
|
1363
|
+
// A shape worth reporting and not worth concluding from: the client is otherwise
|
|
1364
|
+
// indistinguishable from the browser whose headers it copied.
|
|
1365
|
+
verdict: "unknown",
|
|
1366
|
+
detectors: ["challenge-integrity"]
|
|
1367
|
+
},
|
|
1368
|
+
notes: "The proof-of-work floor is measured on the server between issuing and receiving, so no client clock is involved, and it is set at a SHA-256 rate no browser has ever reached. Both signals stay `moderate`: they say the answers did not come from the page we served, which is a fact about the answering software rather than proof about the traffic it is attached to."
|
|
1369
|
+
}),
|
|
1370
|
+
bot({
|
|
1371
|
+
id: "head-only-visit",
|
|
1372
|
+
title: "A visit made entirely of HEAD, claiming a browser",
|
|
1373
|
+
audience: "unwanted-bot",
|
|
1374
|
+
category: "scraping",
|
|
1375
|
+
provenance: "Checking what exists without reading any of it: link checkers, availability monitors and inventory watchers all do this, and a browser navigating never does.",
|
|
1376
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.63", method: "HEAD" }, 30, 900, (index) => `/${["news", "about", "blog", "help", "terms"][index % 5]}`),
|
|
1377
|
+
expect: {
|
|
1378
|
+
// As above: a shape worth reporting, not worth concluding from alone.
|
|
1379
|
+
verdict: "unknown",
|
|
1380
|
+
detectors: ["transport-coherence"]
|
|
1381
|
+
},
|
|
1382
|
+
notes: "One HEAD is a browser checking a link it is about to follow, or a cache revalidating; the shape only means anything across a visit, which is why it is counted on the actor rather than on the request. A link checker is a real and mostly harmless thing to be, so this stays `moderate`."
|
|
1383
|
+
}),
|
|
1384
|
+
bot({
|
|
1385
|
+
id: "catalogue-sweep-by-page",
|
|
1386
|
+
title: "A catalogue taken a page at a time, with the path never changing",
|
|
1387
|
+
audience: "unwanted-bot",
|
|
1388
|
+
category: "scraping",
|
|
1389
|
+
provenance: "How a catalogue is actually taken. The collector copies a browser's headers exactly and walks ?page=1..N, which leaves the path constant \u2014 so distinct-path breadth reads it as somebody rereading one page rather than as enumeration.",
|
|
1390
|
+
requests: repeat({ ...browser("chromeWindows"), ip: "198.51.100.61" }, 40, 900, (index) => `/products?page=${index}`),
|
|
1391
|
+
expect: {
|
|
1392
|
+
// Not proven, and not even suspected at this pace. Said plainly because it is true:
|
|
1393
|
+
// headers this clean leave only behaviour, behaviour is weak by construction, and a
|
|
1394
|
+
// collector polite enough to space its requests stays under the line. What changed
|
|
1395
|
+
// is that it no longer scores *lower* than the identical crawl expressed as distinct
|
|
1396
|
+
// paths — measured at a faster pace before this detector existed, the two differed by
|
|
1397
|
+
// seven points and only the path version crossed; they now score the same at every
|
|
1398
|
+
// volume tried.
|
|
1399
|
+
verdict: "unknown",
|
|
1400
|
+
detectors: ["parameter-sweep"]
|
|
1401
|
+
},
|
|
1402
|
+
notes: "The counterpart to crawl-breadth rather than a replacement for it: breadth counts paths, this counts what is hung on them. Both stay weak, and both are worth having because a collector picks one shape or the other and nothing says which. Neither is a reason to deny anybody on its own."
|
|
1403
|
+
}),
|
|
1148
1404
|
// ---------------------------------------------------------------------------
|
|
1149
1405
|
// Forged identities. The narrow case where a lie is provable.
|
|
1150
1406
|
// ---------------------------------------------------------------------------
|
|
@@ -1487,6 +1743,38 @@ var ADVERSARIAL_CASES = [
|
|
|
1487
1743
|
expect: { certain: false, detectors: ["probe-signature"], neverAction: ["block", "drop"] },
|
|
1488
1744
|
tags: ["scanning"]
|
|
1489
1745
|
}),
|
|
1746
|
+
bot({
|
|
1747
|
+
id: "traversal-encoded-past-a-filter",
|
|
1748
|
+
title: "A traversal with its dots and slashes written in percent-encoding",
|
|
1749
|
+
audience: "hostile",
|
|
1750
|
+
category: "wordlist-probe",
|
|
1751
|
+
provenance: "The standard first move against a path filter, and the reason this library keeps the raw target: normalisation resolves the dots, so what reaches a wordlist check is `/app/config.yml` \u2014 an ordinary-looking path nobody has, on no list. The spelling is the whole signal, and it is destroyed by the thing that makes rules work.",
|
|
1752
|
+
requests: [{ ...browser("chromeWindows"), path: "/%2e%2e%2f%2e%2e%2fapp/config.yml", status: 404 }],
|
|
1753
|
+
expect: { certain: false, detectors: ["target-integrity"], neverAction: ["block", "drop"] },
|
|
1754
|
+
notes: "`strong`, not proven. A path segment carrying a URL as data is encoded to sit in a path and encoded again by whatever built the link, which produces the same characters honestly \u2014 so this may score, and may not close a door on its own.",
|
|
1755
|
+
tags: ["scanning"]
|
|
1756
|
+
}),
|
|
1757
|
+
bot({
|
|
1758
|
+
id: "traversal-double-encoded",
|
|
1759
|
+
title: "A traversal encoded twice, so one round of decoding leaves it encoded",
|
|
1760
|
+
audience: "hostile",
|
|
1761
|
+
category: "wordlist-probe",
|
|
1762
|
+
provenance: "Aimed at a filter that decodes once and then inspects: after its single pass the target still reads `%2e%2e%2f`, which the filter does not recognise, and the server behind it decodes again.",
|
|
1763
|
+
requests: [{ ...browser("chromeWindows"), path: "/static/%252e%252e%252f%252e%252e%252fetc/passwd", status: 404 }],
|
|
1764
|
+
expect: { certain: false, detectors: ["target-integrity"], neverAction: ["block", "drop"] },
|
|
1765
|
+
tags: ["scanning"]
|
|
1766
|
+
}),
|
|
1767
|
+
bot({
|
|
1768
|
+
id: "absolute-form-proxy-probe",
|
|
1769
|
+
title: "A request target addressed to somewhere else entirely",
|
|
1770
|
+
audience: "hostile",
|
|
1771
|
+
category: "protocol-abuse",
|
|
1772
|
+
provenance: "Absolute-form is the request line a client sends to a *proxy*. Arriving at an origin server it is a question \u2014 will you fetch this for me \u2014 and open-proxy scanning asks it of everything with a port 80 open.",
|
|
1773
|
+
requests: [{ ...plain(CHROME_UA), path: "http://scanner.example/check", status: 404 }],
|
|
1774
|
+
expect: { certain: false, detectors: ["target-integrity"], neverAction: ["block", "drop"] },
|
|
1775
|
+
notes: "RFC 9112 \xA73.2.2 requires servers to accept absolute-form, so this is not malformed and is not proven. No browser has ever sent one to an origin server.",
|
|
1776
|
+
tags: ["scanning"]
|
|
1777
|
+
}),
|
|
1490
1778
|
bot({
|
|
1491
1779
|
id: "probe-trace-method",
|
|
1492
1780
|
title: "A TRACE request",
|
|
@@ -3067,6 +3355,46 @@ var HUMAN_CASES = [
|
|
|
3067
3355
|
],
|
|
3068
3356
|
expect: { verdict: "unknown", maxScore: 0, notDetectors: ["header-order"], action: "allow" }
|
|
3069
3357
|
}),
|
|
3358
|
+
human({
|
|
3359
|
+
id: "request-desktop-site-mid-visit",
|
|
3360
|
+
title: "Somebody switching their phone to the desktop version of a site",
|
|
3361
|
+
category: "mangled-by-infrastructure",
|
|
3362
|
+
provenance: "`Request desktop site` rewrites the User-Agent to claim a Mac. The browser is the same Safari and the cookie jar is the same jar, so the marker comes back \u2014 which means the library can see, correctly, that one client has now described itself two different ways.",
|
|
3363
|
+
requires: ["marker-probe"],
|
|
3364
|
+
notes: "The reason `identity-drift` weighs a changed *browser family* at `strong` and a changed platform only at `moderate`. Software does not change what it is; a platform changes when a person taps a menu item, and this is that person.",
|
|
3365
|
+
requests: [
|
|
3366
|
+
...humanPaced({ ...browser("safariIos"), ip: "203.0.115.24", headers: [...browser("safariIos").headers, ["Cookie", "sid=phone-session"]] }, [
|
|
3367
|
+
"/",
|
|
3368
|
+
"/collections/lamps",
|
|
3369
|
+
"/products/brass-desk-lamp",
|
|
3370
|
+
"/products/brass-desk-lamp/reviews"
|
|
3371
|
+
]),
|
|
3372
|
+
// The same person, same session, having tapped "Request desktop site".
|
|
3373
|
+
...humanPaced(
|
|
3374
|
+
{ ...browser("safariMac"), ip: "203.0.115.24", headers: [...browser("safariMac").headers, ["Cookie", "sid=phone-session"]] },
|
|
3375
|
+
["/products/brass-desk-lamp", "/delivery", "/products/brass-desk-lamp", "/basket"]
|
|
3376
|
+
).map((request) => ({ ...request, atMs: (request.atMs ?? 0) + 47e3 }))
|
|
3377
|
+
],
|
|
3378
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
3379
|
+
tags: ["known-cost"]
|
|
3380
|
+
}),
|
|
3381
|
+
human({
|
|
3382
|
+
id: "broken-link-shared-widely",
|
|
3383
|
+
title: "A crowd of people following one mistyped link",
|
|
3384
|
+
category: "mangled-by-infrastructure",
|
|
3385
|
+
provenance: "Somebody shares a URL with a typo in it and thousands of real people follow it within the hour. From the server this is a path the site has never served, requested by many unrelated clients, and answered `not found` to every one of them \u2014 which is the exact shape `path-campaign` reads.",
|
|
3386
|
+
requires: ["site-baseline"],
|
|
3387
|
+
notes: "The known cost of comparing a client with the rest of the traffic: most of the evidence is about what *other* people did, and a person following a bad link is indistinguishable here from one running a list. It is capped at `moderate` for this case specifically, and the guarantee it must keep is this one \u2014 reported, never refused.",
|
|
3388
|
+
requests: Array.from({ length: 16 }, (_, index) => ({
|
|
3389
|
+
...browser("chromeWindows", { kind: "cross-site-navigate", referer: "https://social.example/" }),
|
|
3390
|
+
ip: `203.0.114.${index + 1}`,
|
|
3391
|
+
path: "/blog/anouncing-our-new-thing",
|
|
3392
|
+
status: 404,
|
|
3393
|
+
atMs: index * 4e3
|
|
3394
|
+
})),
|
|
3395
|
+
expect: { certain: false, action: ["allow", "tag", "log", "delay", "challenge", "rate-limit"] },
|
|
3396
|
+
tags: ["known-cost"]
|
|
3397
|
+
}),
|
|
3070
3398
|
human({
|
|
3071
3399
|
id: "cgnat-shared-address",
|
|
3072
3400
|
title: "Many people behind one carrier-grade NAT address",
|
|
@@ -3943,6 +4271,23 @@ function checkExpectations(item, result, assertActions) {
|
|
|
3943
4271
|
}
|
|
3944
4272
|
return failures;
|
|
3945
4273
|
}
|
|
4274
|
+
function withExtraCookies(request, extra) {
|
|
4275
|
+
const headers = [];
|
|
4276
|
+
let merged = false;
|
|
4277
|
+
for (const [name, value] of request.headers) {
|
|
4278
|
+
if (!merged && name.toLowerCase() === "cookie") {
|
|
4279
|
+
headers.push([name, [value, ...extra].join("; ")]);
|
|
4280
|
+
merged = true;
|
|
4281
|
+
} else {
|
|
4282
|
+
headers.push([name, value]);
|
|
4283
|
+
}
|
|
4284
|
+
}
|
|
4285
|
+
if (!merged) headers.push(["Cookie", extra.join("; ")]);
|
|
4286
|
+
return { ...request, headers };
|
|
4287
|
+
}
|
|
4288
|
+
function keepsCookies(request) {
|
|
4289
|
+
return request.headers.some(([name]) => name.toLowerCase() === "cookie");
|
|
4290
|
+
}
|
|
3946
4291
|
async function runCase(handler, clock, item, startedAt, provides, assertActions = true) {
|
|
3947
4292
|
const missing = (item.requires ?? []).filter((capability) => !provides.has(capability));
|
|
3948
4293
|
if (missing.length > 0) {
|
|
@@ -3965,11 +4310,24 @@ async function runCase(handler, clock, item, startedAt, provides, assertActions
|
|
|
3965
4310
|
const seed = toFacts(item.requests[0], fallbackIp, clock.now());
|
|
3966
4311
|
clearanceCookie = handler.grantClearance(seed, item.clearance)?.split(";")[0];
|
|
3967
4312
|
}
|
|
4313
|
+
if (item.challengeHistory !== void 0) {
|
|
4314
|
+
clock.set(startedAt);
|
|
4315
|
+
const seed = toFacts(item.requests[0], fallbackIp, clock.now());
|
|
4316
|
+
const key = handler.actorKeyFor(seed);
|
|
4317
|
+
const state = handler.registry.observe(key, seed);
|
|
4318
|
+
for (let i = 0; i < (item.challengeHistory.replayedSolutions ?? 0); i++) state.noteChallengeAnomaly("replay");
|
|
4319
|
+
for (let i = 0; i < (item.challengeHistory.implausibleSolves ?? 0); i++) state.noteChallengeAnomaly("implausible-speed");
|
|
4320
|
+
}
|
|
4321
|
+
let issuedCookies;
|
|
3968
4322
|
for (const request of item.requests) {
|
|
3969
4323
|
clock.set(startedAt + (request.atMs ?? 0));
|
|
3970
|
-
const
|
|
3971
|
-
const
|
|
4324
|
+
const extraCookies = [clearanceCookie, item.keepsCookies ?? keepsCookies(request) ? issuedCookies : void 0].filter((value) => value !== void 0);
|
|
4325
|
+
const withCookies = extraCookies.length === 0 ? request : withExtraCookies(request, extraCookies);
|
|
4326
|
+
const facts = toFacts(withCookies, fallbackIp, clock.now());
|
|
3972
4327
|
const { assessment, decision, outcome } = await handler.handle(facts);
|
|
4328
|
+
const setCookie = outcome.kind === "continue" ? outcome.responseHeaders?.["set-cookie"] : outcome.kind === "respond" ? outcome.headers["set-cookie"] : void 0;
|
|
4329
|
+
if (setCookie !== void 0) issuedCookies = setCookie.split(";")[0];
|
|
4330
|
+
handler.recordOutcome(facts, request.status ?? 200);
|
|
3973
4331
|
requests.push({ assessment, decision, outcome });
|
|
3974
4332
|
}
|
|
3975
4333
|
const final = requests[requests.length - 1];
|