@riavzon/bot-detector 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +512 -0
- package/dist/_data-sources/suffix.json +98 -0
- package/dist/chunk-C0xms8kb.cjs +34 -0
- package/dist/index.mjs +776 -0
- package/dist/index.mjs.map +1 -0
- package/dist/main.cjs +3236 -0
- package/dist/main.cjs.map +1 -0
- package/dist/main.d.cts +1606 -0
- package/dist/main.d.cts.map +1 -0
- package/dist/main.d.mts +1607 -0
- package/dist/main.d.mts.map +1 -0
- package/dist/main.mjs +3210 -0
- package/dist/main.mjs.map +1 -0
- package/dist/mysqlPoolConnector-9PtXF5E7.mjs +49 -0
- package/dist/mysqlPoolConnector-9PtXF5E7.mjs.map +1 -0
- package/dist/mysqlPoolConnector-D1eVEhRK.cjs +51 -0
- package/dist/mysqlPoolConnector-D1eVEhRK.cjs.map +1 -0
- package/package.json +104 -0
package/dist/main.mjs
ADDED
|
@@ -0,0 +1,3210 @@
|
|
|
1
|
+
import { createConfigManager } from "@riavzon/utils";
|
|
2
|
+
import z from "zod";
|
|
3
|
+
import maxmind from "maxmind";
|
|
4
|
+
import { open } from "lmdb";
|
|
5
|
+
import fs, { existsSync } from "node:fs";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { pino } from "pino";
|
|
9
|
+
import { existsSync as existsSync$1, mkdirSync } from "fs";
|
|
10
|
+
import path$1 from "path";
|
|
11
|
+
import { createStorage } from "unstorage";
|
|
12
|
+
import memoryDriver from "unstorage/drivers/memory";
|
|
13
|
+
import { createDatabase } from "db0";
|
|
14
|
+
import consola from "consola";
|
|
15
|
+
import { randomBytes, randomUUID } from "crypto";
|
|
16
|
+
import { UAParser } from "ua-parser-js";
|
|
17
|
+
import { isAIBot, isBot } from "ua-parser-js/helpers";
|
|
18
|
+
import { CLIs, Crawlers, Fetchers, Libraries } from "ua-parser-js/extensions";
|
|
19
|
+
import { isIP } from "node:net";
|
|
20
|
+
import dns from "node:dns/promises";
|
|
21
|
+
import { anyOf, createRegExp, digit, exactly, letter, maybe } from "magic-regexp";
|
|
22
|
+
import { URL as URL$1 } from "url";
|
|
23
|
+
import { spawn } from "child_process";
|
|
24
|
+
import { performance } from "perf_hooks";
|
|
25
|
+
import { Router } from "express";
|
|
26
|
+
import { compiler } from "@riavzon/shield-base";
|
|
27
|
+
|
|
28
|
+
//#region src/botDetector/types/configSchema.ts
|
|
29
|
+
const dbConfigStorage = z.custom((val) => {
|
|
30
|
+
if (typeof val !== "object" || val === null) return false;
|
|
31
|
+
return typeof val.driver === "string";
|
|
32
|
+
}, { message: "Database config must be an object with a valid \"driver\" string" });
|
|
33
|
+
const store = z.object({ main: dbConfigStorage }).required().strict();
|
|
34
|
+
const cache = z.custom((val) => {
|
|
35
|
+
if (typeof val !== "object" || val === null) return false;
|
|
36
|
+
return typeof val.driver === "string";
|
|
37
|
+
}, { message: "Storage must be an object with a valid \"driver\" string" });
|
|
38
|
+
const configSchema = z.object({
|
|
39
|
+
store,
|
|
40
|
+
banScore: z.number().max(100).min(0).default(100),
|
|
41
|
+
maxScore: z.number().max(100).min(0).default(100),
|
|
42
|
+
restoredReputationPoints: z.number().default(10),
|
|
43
|
+
setNewComputedScore: z.boolean().default(false),
|
|
44
|
+
whiteList: z.array(z.union([
|
|
45
|
+
z.ipv4(),
|
|
46
|
+
z.ipv6(),
|
|
47
|
+
z.string()
|
|
48
|
+
])).optional().default([]),
|
|
49
|
+
checksTimeRateControl: z.object({
|
|
50
|
+
checkEveryRequest: z.boolean().default(true),
|
|
51
|
+
checkEvery: z.number().default(1e3 * 60 * 5)
|
|
52
|
+
}).prefault({}),
|
|
53
|
+
batchQueue: z.object({
|
|
54
|
+
flushIntervalMs: z.number().default(5e3),
|
|
55
|
+
maxBufferSize: z.number().default(100),
|
|
56
|
+
maxRetries: z.number().default(3)
|
|
57
|
+
}).prefault({}),
|
|
58
|
+
storage: cache.optional(),
|
|
59
|
+
checkers: z.object({
|
|
60
|
+
localeMapsCheck: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
61
|
+
enable: z.literal(true),
|
|
62
|
+
penalties: z.object({
|
|
63
|
+
ipAndHeaderMismatch: z.number().default(20),
|
|
64
|
+
missingHeader: z.number().default(20),
|
|
65
|
+
missingGeoData: z.number().default(20),
|
|
66
|
+
malformedHeader: z.number().default(30)
|
|
67
|
+
}).prefault({})
|
|
68
|
+
})]).prefault({ enable: true }),
|
|
69
|
+
knownBadUserAgents: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
70
|
+
enable: z.literal(true),
|
|
71
|
+
penalties: z.object({
|
|
72
|
+
criticalSeverity: z.number().default(100),
|
|
73
|
+
highSeverity: z.number().default(80),
|
|
74
|
+
mediumSeverity: z.number().default(30),
|
|
75
|
+
lowSeverity: z.number().default(10)
|
|
76
|
+
}).prefault({})
|
|
77
|
+
})]).prefault({ enable: true }),
|
|
78
|
+
enableIpChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
79
|
+
enable: z.literal(true),
|
|
80
|
+
penalties: z.number().default(10)
|
|
81
|
+
})]).prefault({ enable: true }),
|
|
82
|
+
enableGoodBotsChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
83
|
+
enable: z.literal(true),
|
|
84
|
+
banUnlistedBots: z.boolean().default(true),
|
|
85
|
+
penalties: z.number().default(100)
|
|
86
|
+
})]).prefault({ enable: true }),
|
|
87
|
+
enableBehaviorRateCheck: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
88
|
+
enable: z.literal(true),
|
|
89
|
+
behavioral_window: z.number().default(6e4),
|
|
90
|
+
behavioral_threshold: z.number().default(30),
|
|
91
|
+
penalties: z.number().default(60)
|
|
92
|
+
})]).prefault({ enable: true }),
|
|
93
|
+
enableProxyIspCookiesChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
94
|
+
enable: z.literal(true),
|
|
95
|
+
penalties: z.object({
|
|
96
|
+
cookieMissing: z.number().default(80),
|
|
97
|
+
proxyDetected: z.number().default(40),
|
|
98
|
+
multiSourceBonus2to3: z.number().default(10),
|
|
99
|
+
multiSourceBonus4plus: z.number().default(20),
|
|
100
|
+
hostingDetected: z.number().default(50),
|
|
101
|
+
ispUnknown: z.number().default(10),
|
|
102
|
+
orgUnknown: z.number().default(10)
|
|
103
|
+
}).prefault({})
|
|
104
|
+
})]).prefault({ enable: true }),
|
|
105
|
+
enableUaAndHeaderChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
106
|
+
enable: z.literal(true),
|
|
107
|
+
penalties: z.object({
|
|
108
|
+
headlessBrowser: z.number().default(100),
|
|
109
|
+
shortUserAgent: z.number().default(80),
|
|
110
|
+
tlsCheckFailed: z.number().default(60),
|
|
111
|
+
badUaChecker: z.boolean().default(true)
|
|
112
|
+
}).prefault({})
|
|
113
|
+
})]).prefault({ enable: true }),
|
|
114
|
+
enableBrowserAndDeviceChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
115
|
+
enable: z.literal(true),
|
|
116
|
+
penalties: z.object({
|
|
117
|
+
cliOrLibrary: z.number().default(100),
|
|
118
|
+
internetExplorer: z.number().default(100),
|
|
119
|
+
linuxOs: z.number().default(10),
|
|
120
|
+
impossibleBrowserCombinations: z.number().default(30),
|
|
121
|
+
browserTypeUnknown: z.number().default(10),
|
|
122
|
+
browserNameUnknown: z.number().default(10),
|
|
123
|
+
desktopWithoutOS: z.number().default(10),
|
|
124
|
+
deviceVendorUnknown: z.number().default(10),
|
|
125
|
+
browserVersionUnknown: z.number().default(10),
|
|
126
|
+
deviceModelUnknown: z.number().default(5)
|
|
127
|
+
}).prefault({})
|
|
128
|
+
})]).prefault({ enable: true }),
|
|
129
|
+
enableGeoChecks: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
130
|
+
enable: z.literal(true),
|
|
131
|
+
bannedCountries: z.array(z.string()).optional().default([]),
|
|
132
|
+
penalties: z.object({
|
|
133
|
+
countryUnknown: z.number().default(10),
|
|
134
|
+
regionUnknown: z.number().default(10),
|
|
135
|
+
latLonUnknown: z.number().default(10),
|
|
136
|
+
districtUnknown: z.number().default(10),
|
|
137
|
+
cityUnknown: z.number().default(10),
|
|
138
|
+
timezoneUnknown: z.number().default(10),
|
|
139
|
+
subregionUnknown: z.number().default(10),
|
|
140
|
+
phoneUnknown: z.number().default(10),
|
|
141
|
+
continentUnknown: z.number().default(10)
|
|
142
|
+
}).prefault({})
|
|
143
|
+
})]).prefault({ enable: true }),
|
|
144
|
+
enableKnownThreatsDetections: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
145
|
+
enable: z.literal(true),
|
|
146
|
+
penalties: z.object({
|
|
147
|
+
anonymiseNetwork: z.number().default(20),
|
|
148
|
+
threatLevels: z.object({
|
|
149
|
+
criticalLevel1: z.number().default(40),
|
|
150
|
+
currentAttacksLevel2: z.number().default(30),
|
|
151
|
+
threatLevel3: z.number().default(20),
|
|
152
|
+
threatLevel4: z.number().default(10)
|
|
153
|
+
}).prefault({})
|
|
154
|
+
}).prefault({})
|
|
155
|
+
})]).prefault({ enable: true }),
|
|
156
|
+
enableAsnClassification: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
157
|
+
enable: z.literal(true),
|
|
158
|
+
penalties: z.object({
|
|
159
|
+
contentClassification: z.number().default(20),
|
|
160
|
+
unknownClassification: z.number().default(10),
|
|
161
|
+
lowVisibilityPenalty: z.number().default(10),
|
|
162
|
+
lowVisibilityThreshold: z.number().default(15),
|
|
163
|
+
comboHostingLowVisibility: z.number().default(20)
|
|
164
|
+
}).prefault({})
|
|
165
|
+
})]).prefault({ enable: true }),
|
|
166
|
+
enableTorAnalysis: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
167
|
+
enable: z.literal(true),
|
|
168
|
+
penalties: z.object({
|
|
169
|
+
runningNode: z.number().default(15),
|
|
170
|
+
exitNode: z.number().default(20),
|
|
171
|
+
webExitCapable: z.number().default(15),
|
|
172
|
+
guardNode: z.number().default(10),
|
|
173
|
+
badExit: z.number().default(40),
|
|
174
|
+
obsoleteVersion: z.number().default(10)
|
|
175
|
+
}).prefault({})
|
|
176
|
+
})]).prefault({ enable: true }),
|
|
177
|
+
enableTimezoneConsistency: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
178
|
+
enable: z.literal(true),
|
|
179
|
+
penalties: z.number().default(20)
|
|
180
|
+
})]).prefault({ enable: true }),
|
|
181
|
+
honeypot: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
182
|
+
enable: z.literal(true),
|
|
183
|
+
paths: z.array(z.string()).default([])
|
|
184
|
+
})]).prefault({ enable: true }),
|
|
185
|
+
enableSessionCoherence: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
186
|
+
enable: z.literal(true),
|
|
187
|
+
penalties: z.object({
|
|
188
|
+
pathMismatch: z.number().default(10),
|
|
189
|
+
missingReferer: z.number().default(20),
|
|
190
|
+
domainMismatch: z.number().default(30)
|
|
191
|
+
}).prefault({})
|
|
192
|
+
})]).prefault({ enable: true }),
|
|
193
|
+
enableVelocityFingerprint: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
194
|
+
enable: z.literal(true),
|
|
195
|
+
cvThreshold: z.number().default(.1),
|
|
196
|
+
penalties: z.number().default(40)
|
|
197
|
+
})]).prefault({ enable: true }),
|
|
198
|
+
enableKnownBadIpsCheck: z.discriminatedUnion("enable", [z.object({ enable: z.literal(false) }), z.object({
|
|
199
|
+
enable: z.literal(true),
|
|
200
|
+
highRiskPenalty: z.number().default(30)
|
|
201
|
+
})]).prefault({ enable: true })
|
|
202
|
+
}),
|
|
203
|
+
generator: z.object({
|
|
204
|
+
scoreThreshold: z.number().default(70),
|
|
205
|
+
generateTypes: z.boolean().default(false),
|
|
206
|
+
deleteAfterBuild: z.boolean().default(false),
|
|
207
|
+
mmdbctlPath: z.string().default("mmdbctl")
|
|
208
|
+
}).prefault({}),
|
|
209
|
+
headerOptions: z.object({
|
|
210
|
+
weightPerMustHeader: z.number().default(20),
|
|
211
|
+
missingBrowserEngine: z.number().default(30),
|
|
212
|
+
postManOrInsomiaHeaders: z.number().default(50),
|
|
213
|
+
AJAXHeaderExists: z.number().default(30),
|
|
214
|
+
connectionHeaderIsClose: z.number().default(20),
|
|
215
|
+
originHeaderIsNULL: z.number().default(10),
|
|
216
|
+
originHeaderMismatch: z.number().default(30),
|
|
217
|
+
omittedAcceptHeader: z.number().default(30),
|
|
218
|
+
clientHintsMissingForBlink: z.number().default(30),
|
|
219
|
+
teHeaderUnexpectedForBlink: z.number().default(10),
|
|
220
|
+
clientHintsUnexpectedForGecko: z.number().default(30),
|
|
221
|
+
teHeaderMissingForGecko: z.number().default(20),
|
|
222
|
+
aggressiveCacheControlOnGet: z.number().default(15),
|
|
223
|
+
crossSiteRequestMissingReferer: z.number().default(10),
|
|
224
|
+
inconsistentSecFetchMode: z.number().default(20),
|
|
225
|
+
hostMismatchWeight: z.number().default(40)
|
|
226
|
+
}).prefault({}),
|
|
227
|
+
pathTraveler: z.object({
|
|
228
|
+
maxIterations: z.number().default(3),
|
|
229
|
+
maxPathLength: z.number().default(1500),
|
|
230
|
+
pathLengthToLong: z.number().default(100),
|
|
231
|
+
longDecoding: z.number().default(100),
|
|
232
|
+
traversalDetected: z.number().default(60)
|
|
233
|
+
}).prefault({}),
|
|
234
|
+
punishmentType: z.object({ enableFireWallBan: z.boolean().default(false) }).prefault({}),
|
|
235
|
+
logLevel: z.enum([
|
|
236
|
+
"debug",
|
|
237
|
+
"info",
|
|
238
|
+
"warn",
|
|
239
|
+
"error",
|
|
240
|
+
"fatal"
|
|
241
|
+
]).default("info")
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
//#endregion
|
|
245
|
+
//#region src/botDetector/db/findDataPath.ts
|
|
246
|
+
const __moduleDir = path.dirname(fileURLToPath(import.meta.url));
|
|
247
|
+
function getLibraryRoot(currentDir = __moduleDir) {
|
|
248
|
+
if (fs.existsSync(path.join(currentDir, "package.json"))) return currentDir;
|
|
249
|
+
const parentDir = path.resolve(currentDir, "..");
|
|
250
|
+
if (parentDir === currentDir) throw new Error("Could not find library root");
|
|
251
|
+
return getLibraryRoot(parentDir);
|
|
252
|
+
}
|
|
253
|
+
function resolveDataPath(fileName) {
|
|
254
|
+
const root = getLibraryRoot();
|
|
255
|
+
const possiblePaths = [path.resolve(root, "_data-sources", fileName), path.resolve(root, "dist", "_data-sources", fileName)];
|
|
256
|
+
for (const path of possiblePaths) if (fs.existsSync(path)) return path;
|
|
257
|
+
if (fileName !== "banned.mmdb" && fileName !== "highRisk.mmdb") throw new Error(`[Bot Detector] Data file "${fileName}" not found. Run 'bot-detector init' to download required data files.`);
|
|
258
|
+
return "";
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
//#endregion
|
|
262
|
+
//#region src/botDetector/helpers/mmdbDataReaders.ts
|
|
263
|
+
var DataSources = class DataSources {
|
|
264
|
+
constructor(readers) {
|
|
265
|
+
this.readers = readers;
|
|
266
|
+
}
|
|
267
|
+
static async initialize() {
|
|
268
|
+
const options = { watchForUpdates: process.env.NODE_ENV !== "test" };
|
|
269
|
+
const [asn, city, country, goodBots, tor, proxy, fireholAnon, fireholLvl1, fireholLvl2, fireholLvl3, fireholLvl4] = await Promise.all([
|
|
270
|
+
maxmind.open(resolveDataPath("asn.mmdb"), options),
|
|
271
|
+
maxmind.open(resolveDataPath("city.mmdb"), options),
|
|
272
|
+
maxmind.open(resolveDataPath("country.mmdb"), options),
|
|
273
|
+
maxmind.open(resolveDataPath("goodBots.mmdb"), options),
|
|
274
|
+
maxmind.open(resolveDataPath("tor.mmdb"), options),
|
|
275
|
+
maxmind.open(resolveDataPath("proxy.mmdb"), options),
|
|
276
|
+
maxmind.open(resolveDataPath("firehol_anonymous.mmdb"), options),
|
|
277
|
+
maxmind.open(resolveDataPath("firehol_l1.mmdb"), options),
|
|
278
|
+
maxmind.open(resolveDataPath("firehol_l2.mmdb"), options),
|
|
279
|
+
maxmind.open(resolveDataPath("firehol_l3.mmdb"), options),
|
|
280
|
+
maxmind.open(resolveDataPath("firehol_l4.mmdb"), options)
|
|
281
|
+
]);
|
|
282
|
+
const bannedPath = resolveDataPath("banned.mmdb");
|
|
283
|
+
const highRiskPath = resolveDataPath("highRisk.mmdb");
|
|
284
|
+
const [banned, highRisk] = await Promise.all([existsSync(bannedPath) ? maxmind.open(bannedPath, options) : Promise.resolve(void 0), existsSync(highRiskPath) ? maxmind.open(highRiskPath, options) : Promise.resolve(void 0)]);
|
|
285
|
+
return new DataSources({
|
|
286
|
+
asn,
|
|
287
|
+
city,
|
|
288
|
+
country,
|
|
289
|
+
goodBots,
|
|
290
|
+
tor,
|
|
291
|
+
proxy,
|
|
292
|
+
fireholAnon,
|
|
293
|
+
fireholLvl1,
|
|
294
|
+
fireholLvl2,
|
|
295
|
+
fireholLvl3,
|
|
296
|
+
fireholLvl4,
|
|
297
|
+
banned,
|
|
298
|
+
highRisk,
|
|
299
|
+
userAgentLmdb: open({
|
|
300
|
+
path: resolveDataPath("useragent-db/useragent.mdb"),
|
|
301
|
+
name: "useragent",
|
|
302
|
+
compression: true,
|
|
303
|
+
readOnly: true,
|
|
304
|
+
useVersions: true,
|
|
305
|
+
sharedStructuresKey: Symbol.for("structures"),
|
|
306
|
+
pageSize: 4096,
|
|
307
|
+
cache: { validated: true },
|
|
308
|
+
noReadAhead: true,
|
|
309
|
+
maxReaders: 2024
|
|
310
|
+
}),
|
|
311
|
+
ja4Lmdb: open({
|
|
312
|
+
path: resolveDataPath("ja4-db/ja4.mdb"),
|
|
313
|
+
name: "ja4",
|
|
314
|
+
compression: true,
|
|
315
|
+
readOnly: true,
|
|
316
|
+
useVersions: true,
|
|
317
|
+
sharedStructuresKey: Symbol.for("structures"),
|
|
318
|
+
pageSize: 4096,
|
|
319
|
+
cache: { validated: true },
|
|
320
|
+
noReadAhead: true,
|
|
321
|
+
maxReaders: 2024
|
|
322
|
+
})
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
asnDataBase(ip) {
|
|
326
|
+
return this.readers.asn.get(ip);
|
|
327
|
+
}
|
|
328
|
+
cityDataBase(ip) {
|
|
329
|
+
return this.readers.city.get(ip);
|
|
330
|
+
}
|
|
331
|
+
countryDataBase(ip) {
|
|
332
|
+
return this.readers.country.get(ip);
|
|
333
|
+
}
|
|
334
|
+
goodBotsDataBase(ip) {
|
|
335
|
+
return this.readers.goodBots.get(ip);
|
|
336
|
+
}
|
|
337
|
+
torDataBase(ip) {
|
|
338
|
+
return this.readers.tor.get(ip);
|
|
339
|
+
}
|
|
340
|
+
proxyDataBase(ip) {
|
|
341
|
+
return this.readers.proxy.get(ip);
|
|
342
|
+
}
|
|
343
|
+
fireholAnonDataBase(ip) {
|
|
344
|
+
return this.readers.fireholAnon.get(ip);
|
|
345
|
+
}
|
|
346
|
+
fireholLvl1DataBase(ip) {
|
|
347
|
+
return this.readers.fireholLvl1.get(ip);
|
|
348
|
+
}
|
|
349
|
+
fireholLvl2DataBase(ip) {
|
|
350
|
+
return this.readers.fireholLvl2.get(ip);
|
|
351
|
+
}
|
|
352
|
+
fireholLvl3DataBase(ip) {
|
|
353
|
+
return this.readers.fireholLvl3.get(ip);
|
|
354
|
+
}
|
|
355
|
+
fireholLvl4DataBase(ip) {
|
|
356
|
+
return this.readers.fireholLvl4.get(ip);
|
|
357
|
+
}
|
|
358
|
+
bannedDataBase(ip) {
|
|
359
|
+
return this.readers.banned?.get(ip) ?? null;
|
|
360
|
+
}
|
|
361
|
+
highRiskDataBase(ip) {
|
|
362
|
+
return this.readers.highRisk?.get(ip) ?? null;
|
|
363
|
+
}
|
|
364
|
+
getUserAgentLmdb() {
|
|
365
|
+
return this.readers.userAgentLmdb;
|
|
366
|
+
}
|
|
367
|
+
getJa4Lmdb() {
|
|
368
|
+
return this.readers.ja4Lmdb;
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
|
|
372
|
+
//#endregion
|
|
373
|
+
//#region src/botDetector/utils/logger.ts
|
|
374
|
+
const LOG_DIR = path$1.resolve(process.cwd(), process.env.LOG_DIR || "bot-detector-logs");
|
|
375
|
+
if (!existsSync$1(LOG_DIR)) mkdirSync(LOG_DIR, { recursive: true });
|
|
376
|
+
const transport = pino.transport({ targets: [
|
|
377
|
+
{
|
|
378
|
+
target: "pino/file",
|
|
379
|
+
level: "info",
|
|
380
|
+
options: {
|
|
381
|
+
destination: `${LOG_DIR}/info.log`,
|
|
382
|
+
mkdir: true
|
|
383
|
+
}
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
target: "pino/file",
|
|
387
|
+
level: "warn",
|
|
388
|
+
options: {
|
|
389
|
+
destination: `${LOG_DIR}/warn.log`,
|
|
390
|
+
mkdir: true
|
|
391
|
+
}
|
|
392
|
+
},
|
|
393
|
+
{
|
|
394
|
+
target: "pino/file",
|
|
395
|
+
level: "error",
|
|
396
|
+
options: {
|
|
397
|
+
destination: `${LOG_DIR}/errors.log`,
|
|
398
|
+
mkdir: true
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
] });
|
|
402
|
+
let logger;
|
|
403
|
+
function getLogger() {
|
|
404
|
+
if (logger) return logger;
|
|
405
|
+
const { logLevel } = getConfiguration();
|
|
406
|
+
logger = pino({
|
|
407
|
+
level: logLevel,
|
|
408
|
+
timestamp: pino.stdTimeFunctions.isoTime,
|
|
409
|
+
mixin() {
|
|
410
|
+
return { uptime: process.uptime() };
|
|
411
|
+
},
|
|
412
|
+
redact: {
|
|
413
|
+
paths: [
|
|
414
|
+
"*.password",
|
|
415
|
+
"*.email",
|
|
416
|
+
"name",
|
|
417
|
+
"Name",
|
|
418
|
+
"*.cookies",
|
|
419
|
+
"*.cookie",
|
|
420
|
+
"cookies",
|
|
421
|
+
"cookie",
|
|
422
|
+
"*.accessToken",
|
|
423
|
+
"*.refresh_token",
|
|
424
|
+
"*.secret"
|
|
425
|
+
],
|
|
426
|
+
censor: "[SECRET]"
|
|
427
|
+
}
|
|
428
|
+
}, transport);
|
|
429
|
+
return logger;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
//#endregion
|
|
433
|
+
//#region src/botDetector/db/dialectUtils.ts
|
|
434
|
+
function isMySQL(db) {
|
|
435
|
+
return db.dialect === "mysql";
|
|
436
|
+
}
|
|
437
|
+
function isSQLite(db) {
|
|
438
|
+
return db.dialect === "sqlite";
|
|
439
|
+
}
|
|
440
|
+
/** Convert ? placeholders to $1, $2, for PostgreSQL. */
|
|
441
|
+
function prep(db, sql) {
|
|
442
|
+
if (db.dialect !== "postgresql") return db.prepare(sql);
|
|
443
|
+
let i = 0;
|
|
444
|
+
return db.prepare(sql.replace(/\?/g, () => `$${String(++i)}`));
|
|
445
|
+
}
|
|
446
|
+
/** Generate positional placeholders for a dynamic list of values. */
|
|
447
|
+
function placeholders(db, count, offset = 0) {
|
|
448
|
+
return Array.from({ length: count }, (_, i) => db.dialect === "postgresql" ? `$${String(offset + i + 1)}` : "?").join(", ");
|
|
449
|
+
}
|
|
450
|
+
/** NOW() equivalent per dialect. */
|
|
451
|
+
function now(db) {
|
|
452
|
+
return isSQLite(db) ? "datetime('now')" : "NOW()";
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* Upsert conflict clause.
|
|
456
|
+
* MySQL: ON DUPLICATE KEY UPDATE
|
|
457
|
+
* Others: ON CONFLICT(pk) DO UPDATE SET
|
|
458
|
+
*/
|
|
459
|
+
function onUpsert(db, pk) {
|
|
460
|
+
return isMySQL(db) ? "ON DUPLICATE KEY UPDATE" : `ON CONFLICT(${pk}) DO UPDATE SET`;
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Reference to the incoming row value in an upsert SET clause.
|
|
464
|
+
* MySQL: VALUES(col)
|
|
465
|
+
* Others: excluded.col
|
|
466
|
+
*/
|
|
467
|
+
function excluded(db, col) {
|
|
468
|
+
return isMySQL(db) ? `VALUES(${col})` : `excluded.${col}`;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
//#endregion
|
|
472
|
+
//#region src/botDetector/db/updateBanned.ts
|
|
473
|
+
async function updateBannedIP(cookie, ipAddress, country, user_agent, info) {
|
|
474
|
+
const db = getDb();
|
|
475
|
+
const log = getLogger().child({
|
|
476
|
+
service: "BOT DETECTOR",
|
|
477
|
+
branch: "db",
|
|
478
|
+
type: "updateBannedIP"
|
|
479
|
+
});
|
|
480
|
+
const params = [
|
|
481
|
+
cookie,
|
|
482
|
+
ipAddress,
|
|
483
|
+
country,
|
|
484
|
+
user_agent,
|
|
485
|
+
JSON.stringify(info.reasons),
|
|
486
|
+
info.score
|
|
487
|
+
];
|
|
488
|
+
const ex = (col) => excluded(db, col);
|
|
489
|
+
const upsert = onUpsert(db, "canary_id");
|
|
490
|
+
try {
|
|
491
|
+
await prep(db, `INSERT INTO banned (canary_id, ip_address, country, user_agent, reason, score)
|
|
492
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
493
|
+
${upsert}
|
|
494
|
+
ip_address = ${ex("ip_address")},
|
|
495
|
+
country = ${ex("country")},
|
|
496
|
+
user_agent = ${ex("user_agent")},
|
|
497
|
+
score = ${ex("score")},
|
|
498
|
+
reason = ${ex("reason")}`).run(...params);
|
|
499
|
+
log.info(`Updated Database TABLE - banned. A user has been banned for IP ${ipAddress} (score ${String(info.score)})`);
|
|
500
|
+
} catch (err) {
|
|
501
|
+
log.error({ error: err }, "ERROR UPDATING \"banned\" TABLE");
|
|
502
|
+
throw err;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
//#endregion
|
|
507
|
+
//#region src/botDetector/db/updateIsBot.ts
|
|
508
|
+
async function updateIsBot(isBot, cookie) {
|
|
509
|
+
const params = [isBot, cookie];
|
|
510
|
+
const db = getDb();
|
|
511
|
+
const log = getLogger().child({
|
|
512
|
+
service: "BOT DETECTOR",
|
|
513
|
+
branch: "db",
|
|
514
|
+
type: "updateIsBot"
|
|
515
|
+
});
|
|
516
|
+
try {
|
|
517
|
+
await prep(db, `UPDATE visitors SET is_bot = ? WHERE canary_id = ?`).run(...params);
|
|
518
|
+
} catch (err) {
|
|
519
|
+
log.error({ error: err }, "ERROR UPDATING IS_BOT");
|
|
520
|
+
throw err;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
//#endregion
|
|
525
|
+
//#region src/botDetector/db/updateVisitors.ts
|
|
526
|
+
async function updateVisitor(u) {
|
|
527
|
+
const db = getDb();
|
|
528
|
+
const log = getLogger().child({
|
|
529
|
+
service: "BOT DETECTOR",
|
|
530
|
+
branch: "db",
|
|
531
|
+
type: "updateVisitors"
|
|
532
|
+
});
|
|
533
|
+
const { visitorId, cookie, ipAddress, userAgent, country, region, regionName, city, district, lat, lon, timezone, currency, isp, org, as: asOrg, device_type, browser, proxy, hosting, is_bot, first_seen, last_seen, request_count, deviceVendor, deviceModel, browserType, browserVersion, os, activity_score } = u;
|
|
534
|
+
const params = [
|
|
535
|
+
visitorId,
|
|
536
|
+
cookie,
|
|
537
|
+
ipAddress,
|
|
538
|
+
userAgent,
|
|
539
|
+
country,
|
|
540
|
+
region,
|
|
541
|
+
regionName,
|
|
542
|
+
city,
|
|
543
|
+
district,
|
|
544
|
+
lat,
|
|
545
|
+
lon,
|
|
546
|
+
timezone,
|
|
547
|
+
currency,
|
|
548
|
+
isp,
|
|
549
|
+
org,
|
|
550
|
+
asOrg,
|
|
551
|
+
device_type,
|
|
552
|
+
browser,
|
|
553
|
+
proxy,
|
|
554
|
+
hosting,
|
|
555
|
+
is_bot,
|
|
556
|
+
first_seen,
|
|
557
|
+
last_seen,
|
|
558
|
+
request_count,
|
|
559
|
+
deviceVendor,
|
|
560
|
+
deviceModel,
|
|
561
|
+
browserType,
|
|
562
|
+
browserVersion,
|
|
563
|
+
os,
|
|
564
|
+
Number(activity_score) || 0
|
|
565
|
+
].map((value) => value === void 0 ? null : value);
|
|
566
|
+
const ex = (col) => excluded(db, col);
|
|
567
|
+
const upsert = onUpsert(db, "canary_id");
|
|
568
|
+
try {
|
|
569
|
+
await prep(db, `INSERT INTO visitors (
|
|
570
|
+
visitor_id,
|
|
571
|
+
canary_id,
|
|
572
|
+
ip_address,
|
|
573
|
+
user_agent,
|
|
574
|
+
country,
|
|
575
|
+
region,
|
|
576
|
+
region_name,
|
|
577
|
+
city,
|
|
578
|
+
district,
|
|
579
|
+
lat,
|
|
580
|
+
lon,
|
|
581
|
+
timezone,
|
|
582
|
+
currency,
|
|
583
|
+
isp,
|
|
584
|
+
org,
|
|
585
|
+
as_org,
|
|
586
|
+
device_type,
|
|
587
|
+
browser,
|
|
588
|
+
proxy,
|
|
589
|
+
hosting,
|
|
590
|
+
is_bot,
|
|
591
|
+
first_seen,
|
|
592
|
+
last_seen,
|
|
593
|
+
request_count,
|
|
594
|
+
deviceVendor,
|
|
595
|
+
deviceModel,
|
|
596
|
+
browserType,
|
|
597
|
+
browserVersion,
|
|
598
|
+
os,
|
|
599
|
+
suspicious_activity_score
|
|
600
|
+
) VALUES (
|
|
601
|
+
${params.map(() => "?").join(", ")}
|
|
602
|
+
)
|
|
603
|
+
${upsert}
|
|
604
|
+
ip_address = ${ex("ip_address")},
|
|
605
|
+
user_agent = ${ex("user_agent")},
|
|
606
|
+
country = ${ex("country")},
|
|
607
|
+
region = ${ex("region")},
|
|
608
|
+
region_name = ${ex("region_name")},
|
|
609
|
+
city = ${ex("city")},
|
|
610
|
+
district = ${ex("district")},
|
|
611
|
+
lat = ${ex("lat")},
|
|
612
|
+
lon = ${ex("lon")},
|
|
613
|
+
timezone = ${ex("timezone")},
|
|
614
|
+
currency = ${ex("currency")},
|
|
615
|
+
isp = ${ex("isp")},
|
|
616
|
+
org = ${ex("org")},
|
|
617
|
+
as_org = ${ex("as_org")},
|
|
618
|
+
device_type = ${ex("device_type")},
|
|
619
|
+
browser = ${ex("browser")},
|
|
620
|
+
proxy = ${ex("proxy")},
|
|
621
|
+
hosting = ${ex("hosting")},
|
|
622
|
+
last_seen = ${now(db)},
|
|
623
|
+
request_count = request_count + 1,
|
|
624
|
+
deviceVendor = ${ex("deviceVendor")},
|
|
625
|
+
deviceModel = ${ex("deviceModel")},
|
|
626
|
+
browserType = ${ex("browserType")},
|
|
627
|
+
browserVersion = ${ex("browserVersion")},
|
|
628
|
+
os = ${ex("os")}`).run(...params);
|
|
629
|
+
log.info(`Updated visitors table, Visitor row for canary_id=${cookie ?? ""} inserted/updated successfully.`);
|
|
630
|
+
return;
|
|
631
|
+
} catch (err) {
|
|
632
|
+
log.error({ error: err }, `ERROR UPDATING visitors TABLE`);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
//#endregion
|
|
637
|
+
//#region src/botDetector/db/updateVisitorScore.ts
|
|
638
|
+
async function updateScore(score, cookie) {
|
|
639
|
+
const params = [score, cookie];
|
|
640
|
+
const db = getDb();
|
|
641
|
+
const log = getLogger().child({
|
|
642
|
+
service: "BOT DETECTOR",
|
|
643
|
+
branch: "db",
|
|
644
|
+
type: "updateScore"
|
|
645
|
+
});
|
|
646
|
+
try {
|
|
647
|
+
await prep(db, `UPDATE visitors SET suspicious_activity_score = ? WHERE canary_id = ?`).run(...params);
|
|
648
|
+
} catch (err) {
|
|
649
|
+
log.error({ error: err }, "ERROR UPDATING SCORE");
|
|
650
|
+
throw err;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
//#endregion
|
|
655
|
+
//#region src/botDetector/db/batchQueue.ts
|
|
656
|
+
var BatchQueue = class {
|
|
657
|
+
constructor() {
|
|
658
|
+
this.jobs = /* @__PURE__ */ new Map();
|
|
659
|
+
this.timer = null;
|
|
660
|
+
this.flushPromise = null;
|
|
661
|
+
}
|
|
662
|
+
get config() {
|
|
663
|
+
return getConfiguration().batchQueue;
|
|
664
|
+
}
|
|
665
|
+
get log() {
|
|
666
|
+
return getLogger().child({
|
|
667
|
+
service: "BOT DETECTOR",
|
|
668
|
+
branch: "BatchQueue"
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
async addQueue(canary, ipAddress, type, params, priority = "deferred") {
|
|
672
|
+
const key = `${type}:${canary}:${ipAddress}`;
|
|
673
|
+
this.jobs.set(key, {
|
|
674
|
+
id: key,
|
|
675
|
+
type,
|
|
676
|
+
priority,
|
|
677
|
+
params
|
|
678
|
+
});
|
|
679
|
+
if (priority === "immediate" || this.jobs.size >= this.config.maxBufferSize) await this.flush();
|
|
680
|
+
else this.timer ??= setTimeout(() => void this.flush(), this.config.flushIntervalMs);
|
|
681
|
+
}
|
|
682
|
+
async flush() {
|
|
683
|
+
while (this.flushPromise || this.jobs.size > 0) {
|
|
684
|
+
if (this.flushPromise) await this.flushPromise;
|
|
685
|
+
if (this.jobs.size > 0) {
|
|
686
|
+
if (this.timer) {
|
|
687
|
+
clearTimeout(this.timer);
|
|
688
|
+
this.timer = null;
|
|
689
|
+
}
|
|
690
|
+
const currentBatch = Array.from(this.jobs.values());
|
|
691
|
+
this.jobs.clear();
|
|
692
|
+
this.flushPromise = this.executeBatch(currentBatch, 0);
|
|
693
|
+
try {
|
|
694
|
+
await this.flushPromise;
|
|
695
|
+
} finally {
|
|
696
|
+
this.flushPromise = null;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
runJob(job) {
|
|
702
|
+
switch (job.type) {
|
|
703
|
+
case "visitor_upsert": return updateVisitor(job.params.insert);
|
|
704
|
+
case "score_update": {
|
|
705
|
+
const { score, cookie } = job.params;
|
|
706
|
+
return updateScore(score, cookie);
|
|
707
|
+
}
|
|
708
|
+
case "is_bot_update": {
|
|
709
|
+
const { isBot, cookie } = job.params;
|
|
710
|
+
return updateIsBot(isBot, cookie);
|
|
711
|
+
}
|
|
712
|
+
case "update_banned_ip": {
|
|
713
|
+
const { cookie, ipAddress, country, user_agent, info } = job.params;
|
|
714
|
+
return updateBannedIP(cookie, ipAddress, country, user_agent, info);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
async executeBatch(batch, retryCount) {
|
|
719
|
+
try {
|
|
720
|
+
const visitors = batch.filter((j) => j.type === "visitor_upsert");
|
|
721
|
+
const others = batch.filter((j) => j.type !== "visitor_upsert");
|
|
722
|
+
if (visitors.length > 0) await Promise.all(visitors.map((j) => this.runJob(j)));
|
|
723
|
+
await Promise.all(others.map((j) => this.runJob(j)));
|
|
724
|
+
} catch (err) {
|
|
725
|
+
this.log.error({ err }, `Batch flush failed (Attempt ${String(retryCount + 1)})`);
|
|
726
|
+
if (retryCount < this.config.maxRetries) {
|
|
727
|
+
await new Promise((res) => setTimeout(res, 1e3));
|
|
728
|
+
return this.executeBatch(batch, retryCount + 1);
|
|
729
|
+
}
|
|
730
|
+
this.log.error("Max retries reached. Discarding batch.");
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
async shutdown() {
|
|
734
|
+
this.log.info("Shutting down BatchQueue: Draining remaining jobs...");
|
|
735
|
+
await this.flush();
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
|
|
739
|
+
//#endregion
|
|
740
|
+
//#region src/botDetector/config/storageAdapter.ts
|
|
741
|
+
async function initStorage(config) {
|
|
742
|
+
if (!config) return createStorage({ driver: memoryDriver() });
|
|
743
|
+
const { driver, ...opts } = config;
|
|
744
|
+
let mod;
|
|
745
|
+
switch (driver) {
|
|
746
|
+
case "redis":
|
|
747
|
+
mod = await import("unstorage/drivers/redis");
|
|
748
|
+
break;
|
|
749
|
+
case "upstash":
|
|
750
|
+
mod = await import("unstorage/drivers/upstash");
|
|
751
|
+
break;
|
|
752
|
+
case "lru":
|
|
753
|
+
mod = await import("unstorage/drivers/lru-cache");
|
|
754
|
+
break;
|
|
755
|
+
case "fs":
|
|
756
|
+
mod = await import("unstorage/drivers/fs-lite");
|
|
757
|
+
break;
|
|
758
|
+
case "cloudflare-kv-binding":
|
|
759
|
+
mod = await import("unstorage/drivers/cloudflare-kv-binding");
|
|
760
|
+
break;
|
|
761
|
+
case "cloudflare-kv-http":
|
|
762
|
+
mod = await import("unstorage/drivers/cloudflare-kv-http");
|
|
763
|
+
break;
|
|
764
|
+
case "cloudflare-r2-binding":
|
|
765
|
+
mod = await import("unstorage/drivers/cloudflare-r2-binding");
|
|
766
|
+
break;
|
|
767
|
+
case "vercel":
|
|
768
|
+
mod = await import("unstorage/drivers/vercel-runtime-cache");
|
|
769
|
+
break;
|
|
770
|
+
default: throw new Error(`Unsupported storage driver: ${driver}`);
|
|
771
|
+
}
|
|
772
|
+
return createStorage({ driver: mod.default(opts) });
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
//#endregion
|
|
776
|
+
//#region src/botDetector/config/dbAdapter.ts
|
|
777
|
+
async function initDb(config) {
|
|
778
|
+
const { driver, ...opts } = config;
|
|
779
|
+
let mod;
|
|
780
|
+
switch (driver) {
|
|
781
|
+
case "mysql-pool":
|
|
782
|
+
mod = await import("./mysqlPoolConnector-9PtXF5E7.mjs");
|
|
783
|
+
break;
|
|
784
|
+
case "postgresql":
|
|
785
|
+
mod = await import("db0/connectors/postgresql");
|
|
786
|
+
break;
|
|
787
|
+
case "sqlite":
|
|
788
|
+
mod = await import("db0/connectors/better-sqlite3");
|
|
789
|
+
break;
|
|
790
|
+
case "cloudflare-d1":
|
|
791
|
+
mod = await import("db0/connectors/cloudflare-d1");
|
|
792
|
+
break;
|
|
793
|
+
case "planetscale":
|
|
794
|
+
mod = await import("db0/connectors/planetscale");
|
|
795
|
+
break;
|
|
796
|
+
default: throw new Error(`Unsupported database driver: ${driver}`);
|
|
797
|
+
}
|
|
798
|
+
return createDatabase(mod.default(opts));
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
//#endregion
|
|
802
|
+
//#region src/botDetector/checkers/CheckerRegistry.ts
|
|
803
|
+
let registeredCheckers = [];
|
|
804
|
+
const CheckerRegistry = {
|
|
805
|
+
register(checker) {
|
|
806
|
+
consola.log(`Loaded plugin: ${checker.name}`);
|
|
807
|
+
registeredCheckers.push(checker);
|
|
808
|
+
},
|
|
809
|
+
getEnabled(phase, config) {
|
|
810
|
+
return registeredCheckers.filter((checker) => checker.phase === phase && checker.isEnabled(config));
|
|
811
|
+
},
|
|
812
|
+
clear() {
|
|
813
|
+
registeredCheckers = [];
|
|
814
|
+
}
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
//#endregion
|
|
818
|
+
//#region src/botDetector/checkers/badUaChecker.ts
|
|
819
|
+
const SEVERITY_ORDER = [
|
|
820
|
+
"critical",
|
|
821
|
+
"high",
|
|
822
|
+
"medium",
|
|
823
|
+
"low"
|
|
824
|
+
];
|
|
825
|
+
let patterns = [];
|
|
826
|
+
function loadUaPatterns() {
|
|
827
|
+
const db = getDataSources().getUserAgentLmdb();
|
|
828
|
+
const bucketStrings = new Map(SEVERITY_ORDER.map((sev) => [sev, []]));
|
|
829
|
+
for (const { value } of db.getRange({ limit: 1e4 })) {
|
|
830
|
+
const sev = value.metadata_severity;
|
|
831
|
+
bucketStrings.get(sev)?.push(value.useragent_rx);
|
|
832
|
+
}
|
|
833
|
+
patterns = SEVERITY_ORDER.map((severity) => {
|
|
834
|
+
const patterns = bucketStrings.get(severity) ?? [];
|
|
835
|
+
if (patterns.length === 0) return null;
|
|
836
|
+
const combined = patterns.map((p) => `(?:${p})`).join("|");
|
|
837
|
+
return {
|
|
838
|
+
rx: new RegExp(combined, "i"),
|
|
839
|
+
severity
|
|
840
|
+
};
|
|
841
|
+
}).filter((p) => p !== null);
|
|
842
|
+
}
|
|
843
|
+
var BadUaChecker = class {
|
|
844
|
+
constructor() {
|
|
845
|
+
this.name = "Bad User Agent list";
|
|
846
|
+
this.phase = "heavy";
|
|
847
|
+
}
|
|
848
|
+
isEnabled(config) {
|
|
849
|
+
return config.checkers.knownBadUserAgents.enable;
|
|
850
|
+
}
|
|
851
|
+
run(ctx, config) {
|
|
852
|
+
const { knownBadUserAgents } = config.checkers;
|
|
853
|
+
const reasons = [];
|
|
854
|
+
let score = 0;
|
|
855
|
+
if (!knownBadUserAgents.enable) return {
|
|
856
|
+
score,
|
|
857
|
+
reasons
|
|
858
|
+
};
|
|
859
|
+
if (patterns.length === 0) loadUaPatterns();
|
|
860
|
+
const rawUa = ctx.req.get("User-Agent") ?? "";
|
|
861
|
+
for (const { rx, severity } of patterns) if (rx.test(rawUa)) {
|
|
862
|
+
reasons.push("BAD_UA_DETECTED");
|
|
863
|
+
switch (severity) {
|
|
864
|
+
case "critical":
|
|
865
|
+
score += knownBadUserAgents.penalties.criticalSeverity;
|
|
866
|
+
break;
|
|
867
|
+
case "high":
|
|
868
|
+
score += knownBadUserAgents.penalties.highSeverity;
|
|
869
|
+
break;
|
|
870
|
+
case "medium":
|
|
871
|
+
score += knownBadUserAgents.penalties.mediumSeverity;
|
|
872
|
+
break;
|
|
873
|
+
case "low":
|
|
874
|
+
score += knownBadUserAgents.penalties.lowSeverity;
|
|
875
|
+
break;
|
|
876
|
+
}
|
|
877
|
+
break;
|
|
878
|
+
}
|
|
879
|
+
return {
|
|
880
|
+
score,
|
|
881
|
+
reasons
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
CheckerRegistry.register(new BadUaChecker());
|
|
886
|
+
|
|
887
|
+
//#endregion
|
|
888
|
+
//#region src/botDetector/config/config.ts
|
|
889
|
+
const { defineConfiguration, getConfiguration } = createConfigManager(configSchema, "Bot Detector");
|
|
890
|
+
let globalDataSources;
|
|
891
|
+
let globalBatchQueue;
|
|
892
|
+
let globalStorage;
|
|
893
|
+
let globalDb;
|
|
894
|
+
/**
|
|
895
|
+
* @description
|
|
896
|
+
* The bot detector library's configuration object.
|
|
897
|
+
* Contains the core configuration to make the library usable client side.
|
|
898
|
+
* @module jwtAuth/config
|
|
899
|
+
* @see {@link ./jwtAuth/types/configSchema.js}
|
|
900
|
+
*/
|
|
901
|
+
async function configuration(config) {
|
|
902
|
+
const initDataSourcesTask = async () => {
|
|
903
|
+
globalDataSources ??= await DataSources.initialize();
|
|
904
|
+
loadUaPatterns();
|
|
905
|
+
};
|
|
906
|
+
const initBatchQueueTask = () => {
|
|
907
|
+
if (!globalBatchQueue) {
|
|
908
|
+
globalBatchQueue = new BatchQueue();
|
|
909
|
+
process.on("SIGTERM", () => {
|
|
910
|
+
globalBatchQueue?.shutdown();
|
|
911
|
+
});
|
|
912
|
+
process.on("SIGINT", () => {
|
|
913
|
+
globalBatchQueue?.shutdown();
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
};
|
|
917
|
+
const initStorageTask = async () => {
|
|
918
|
+
globalStorage ??= await initStorage(config.storage);
|
|
919
|
+
};
|
|
920
|
+
const initDbTask = async () => {
|
|
921
|
+
globalDb ??= await initDb(config.store.main);
|
|
922
|
+
};
|
|
923
|
+
await defineConfiguration(config, [
|
|
924
|
+
initDataSourcesTask,
|
|
925
|
+
initBatchQueueTask,
|
|
926
|
+
initStorageTask,
|
|
927
|
+
initDbTask
|
|
928
|
+
]);
|
|
929
|
+
}
|
|
930
|
+
function getBatchQueue() {
|
|
931
|
+
if (!globalBatchQueue) {
|
|
932
|
+
consola.trace("Premature getBatchQueue() call");
|
|
933
|
+
throw new Error("BatchQueue not ready. Call configuration() first.");
|
|
934
|
+
}
|
|
935
|
+
return globalBatchQueue;
|
|
936
|
+
}
|
|
937
|
+
function getStorage() {
|
|
938
|
+
if (!globalStorage) {
|
|
939
|
+
consola.trace("Premature getStorage() call");
|
|
940
|
+
throw new Error("Storage not ready. Call configuration() first.");
|
|
941
|
+
}
|
|
942
|
+
return globalStorage;
|
|
943
|
+
}
|
|
944
|
+
function getDb() {
|
|
945
|
+
if (!globalDb) {
|
|
946
|
+
consola.trace("Premature getDb() call");
|
|
947
|
+
throw new Error("DB not ready. Call configuration() first.");
|
|
948
|
+
}
|
|
949
|
+
return globalDb;
|
|
950
|
+
}
|
|
951
|
+
function getDataSources() {
|
|
952
|
+
if (!globalDataSources) {
|
|
953
|
+
consola.trace("Premature getDataSources() call");
|
|
954
|
+
throw new Error(`##### Must be initialized globally #####
|
|
955
|
+
Bot Detector: DataSources not ready. Call configuration() first.`);
|
|
956
|
+
}
|
|
957
|
+
return globalDataSources;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
//#endregion
|
|
961
|
+
//#region src/botDetector/helpers/getIPInformation.ts
|
|
962
|
+
const norm = (string) => string?.trim().toLowerCase();
|
|
963
|
+
function getData(ip) {
|
|
964
|
+
const dataSource = getDataSources();
|
|
965
|
+
const countryLvl = dataSource.countryDataBase(ip);
|
|
966
|
+
const cityLvl = dataSource.cityDataBase(ip);
|
|
967
|
+
const asn = dataSource.asnDataBase(ip);
|
|
968
|
+
const proxy = dataSource.proxyDataBase(ip);
|
|
969
|
+
const tor = dataSource.torDataBase(ip);
|
|
970
|
+
return {
|
|
971
|
+
country: norm(countryLvl?.name ?? cityLvl?.name),
|
|
972
|
+
countryCode: norm(countryLvl?.country_code ?? cityLvl?.country_code),
|
|
973
|
+
region: norm(cityLvl?.region ?? countryLvl?.region),
|
|
974
|
+
regionName: norm(cityLvl?.continent ?? cityLvl?.subregion ?? countryLvl?.subregion),
|
|
975
|
+
subregion: norm(cityLvl?.subregion ?? countryLvl?.subregion),
|
|
976
|
+
state: norm(cityLvl?.state),
|
|
977
|
+
zipCode: norm(cityLvl?.zip_code),
|
|
978
|
+
city: norm(cityLvl?.city ?? cityLvl?.capital ?? countryLvl?.capital),
|
|
979
|
+
phone: norm(cityLvl?.phone ?? countryLvl?.phone),
|
|
980
|
+
numericCode: norm(cityLvl?.numericCode ?? countryLvl?.numericCode),
|
|
981
|
+
native: norm(cityLvl?.native ?? countryLvl?.native),
|
|
982
|
+
continent: norm(cityLvl?.continent),
|
|
983
|
+
capital: norm(cityLvl?.capital ?? countryLvl?.capital),
|
|
984
|
+
district: norm(cityLvl?.state),
|
|
985
|
+
lat: norm(cityLvl?.latitude),
|
|
986
|
+
lon: norm(cityLvl?.longitude),
|
|
987
|
+
timezone: norm(cityLvl?.timezone ?? countryLvl?.timezone),
|
|
988
|
+
timeZoneName: norm(cityLvl?.timeZoneName ?? countryLvl?.timeZoneName),
|
|
989
|
+
utc_offset: norm(cityLvl?.utc_offset ?? countryLvl?.utc_offset),
|
|
990
|
+
tld: norm(cityLvl?.tld ?? countryLvl?.tld),
|
|
991
|
+
nationality: norm(cityLvl?.nationality ?? countryLvl?.nationality),
|
|
992
|
+
currency: norm(cityLvl?.currency ?? countryLvl?.currency),
|
|
993
|
+
iso639: norm(cityLvl?.iso639 ?? countryLvl?.iso639),
|
|
994
|
+
languages: norm(cityLvl?.languages ?? countryLvl?.languages),
|
|
995
|
+
isp: norm(asn?.asn_name),
|
|
996
|
+
org: norm(asn?.asn_id),
|
|
997
|
+
as_org: norm(asn?.asn_name),
|
|
998
|
+
proxy: proxy ? true : false,
|
|
999
|
+
hosting: asn?.classification === "Content" || Boolean(tor?.exit_addresses)
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
//#endregion
|
|
1004
|
+
//#region src/botDetector/utils/cookieGenerator.ts
|
|
1005
|
+
function makeCookie(res, name, value, options) {
|
|
1006
|
+
if (name.startsWith("__Host-")) {
|
|
1007
|
+
options.secure = true;
|
|
1008
|
+
options.path = "/";
|
|
1009
|
+
delete options.domain;
|
|
1010
|
+
}
|
|
1011
|
+
if (name.startsWith("__Secure-")) options.secure = true;
|
|
1012
|
+
res.cookie(name, value, {
|
|
1013
|
+
httpOnly: options.httpOnly,
|
|
1014
|
+
sameSite: options.sameSite,
|
|
1015
|
+
maxAge: options.maxAge,
|
|
1016
|
+
secure: options.secure,
|
|
1017
|
+
expires: options.expires,
|
|
1018
|
+
domain: options.domain,
|
|
1019
|
+
path: options.path
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
//#endregion
|
|
1024
|
+
//#region src/botDetector/helpers/UAparser.ts
|
|
1025
|
+
function parseUA(userAgent) {
|
|
1026
|
+
const botParser = new UAParser([
|
|
1027
|
+
Crawlers,
|
|
1028
|
+
CLIs,
|
|
1029
|
+
Fetchers,
|
|
1030
|
+
Libraries
|
|
1031
|
+
]);
|
|
1032
|
+
const uaString = typeof userAgent === "string" ? userAgent : String(userAgent);
|
|
1033
|
+
const result = botParser.setUA(uaString).getResult();
|
|
1034
|
+
return {
|
|
1035
|
+
device: result.device.type ?? "desktop",
|
|
1036
|
+
deviceVendor: result.device.vendor,
|
|
1037
|
+
deviceModel: result.device.model,
|
|
1038
|
+
browser: result.browser.name,
|
|
1039
|
+
browserType: result.browser.type,
|
|
1040
|
+
browserVersion: result.browser.version,
|
|
1041
|
+
os: result.os.name,
|
|
1042
|
+
botAI: isAIBot(result),
|
|
1043
|
+
bot: isBot(result),
|
|
1044
|
+
allResults: result
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
//#endregion
|
|
1049
|
+
//#region src/botDetector/checkers/ipValidation.ts
|
|
1050
|
+
var IpChecker = class {
|
|
1051
|
+
constructor() {
|
|
1052
|
+
this.name = "IP Validation";
|
|
1053
|
+
this.phase = "cheap";
|
|
1054
|
+
}
|
|
1055
|
+
isEnabled(config) {
|
|
1056
|
+
return config.checkers.enableIpChecks.enable;
|
|
1057
|
+
}
|
|
1058
|
+
run(ctx, config) {
|
|
1059
|
+
const isValid = isIP(ctx.ipAddress) !== 0;
|
|
1060
|
+
return {
|
|
1061
|
+
score: isValid ? 0 : config.banScore,
|
|
1062
|
+
reasons: isValid ? [] : ["IP_INVALID"]
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
CheckerRegistry.register(new IpChecker());
|
|
1067
|
+
|
|
1068
|
+
//#endregion
|
|
1069
|
+
//#region src/botDetector/helpers/cache/dnsLookupCache.ts
|
|
1070
|
+
const DNS_TTL_SECONDS = 3600 * 2;
|
|
1071
|
+
const PREFIX$5 = "dns:";
|
|
1072
|
+
const dnsCache = {
|
|
1073
|
+
async get(ip) {
|
|
1074
|
+
return getStorage().getItem(`${PREFIX$5}${ip}`);
|
|
1075
|
+
},
|
|
1076
|
+
async set(ip, entry) {
|
|
1077
|
+
await getStorage().setItem(`${PREFIX$5}${ip}`, entry, { ttl: DNS_TTL_SECONDS });
|
|
1078
|
+
},
|
|
1079
|
+
async delete(ip) {
|
|
1080
|
+
await getStorage().removeItem(`${PREFIX$5}${ip}`);
|
|
1081
|
+
},
|
|
1082
|
+
async clear() {
|
|
1083
|
+
await getStorage().clear();
|
|
1084
|
+
}
|
|
1085
|
+
};
|
|
1086
|
+
|
|
1087
|
+
//#endregion
|
|
1088
|
+
//#region src/botDetector/checkers/goodBots/base.ts
|
|
1089
|
+
var GoodBotsBase = class {
|
|
1090
|
+
getDomains(suffix) {
|
|
1091
|
+
const allDomains = [];
|
|
1092
|
+
for (const key in suffix) {
|
|
1093
|
+
const entrySuffix = suffix[key].suffix;
|
|
1094
|
+
if (Array.isArray(entrySuffix)) allDomains.push(...entrySuffix);
|
|
1095
|
+
else if (typeof entrySuffix === "string") allDomains.push(entrySuffix);
|
|
1096
|
+
}
|
|
1097
|
+
return allDomains;
|
|
1098
|
+
}
|
|
1099
|
+
get logger() {
|
|
1100
|
+
this._logger ??= getLogger().child({
|
|
1101
|
+
service: "botDetector",
|
|
1102
|
+
branch: "checker",
|
|
1103
|
+
type: "GoodBotsBase"
|
|
1104
|
+
});
|
|
1105
|
+
return this._logger;
|
|
1106
|
+
}
|
|
1107
|
+
constructor(suffixes) {
|
|
1108
|
+
this.suffixes = suffixes;
|
|
1109
|
+
this.domains = this.getDomains(this.suffixes).map((d) => `.${d.toLowerCase()}`);
|
|
1110
|
+
}
|
|
1111
|
+
async isBotFromTrustedDomain(ip) {
|
|
1112
|
+
const cached = await dnsCache.get(ip);
|
|
1113
|
+
if (cached) return cached.trustedBot;
|
|
1114
|
+
try {
|
|
1115
|
+
const matchingHosts = (await dns.reverse(ip)).filter((host) => this.domains.some((domain) => host.endsWith(domain)));
|
|
1116
|
+
if (matchingHosts.length === 0) {
|
|
1117
|
+
dnsCache.set(ip, {
|
|
1118
|
+
ip,
|
|
1119
|
+
trustedBot: false
|
|
1120
|
+
}).catch((err) => {
|
|
1121
|
+
this.logger.error({ err }, "Failed to save dnsCache in storage");
|
|
1122
|
+
});
|
|
1123
|
+
return false;
|
|
1124
|
+
}
|
|
1125
|
+
for (const host of matchingHosts) if ((await dns.lookup(host, { all: true })).some((a) => a.address === ip)) {
|
|
1126
|
+
dnsCache.set(ip, {
|
|
1127
|
+
ip,
|
|
1128
|
+
trustedBot: true
|
|
1129
|
+
}).catch((err) => {
|
|
1130
|
+
this.logger.error({ err }, "Failed to save dnsCache in storage");
|
|
1131
|
+
});
|
|
1132
|
+
return true;
|
|
1133
|
+
}
|
|
1134
|
+
} catch (err) {
|
|
1135
|
+
this.logger.error({ err }, "DNS reverse lookup failed");
|
|
1136
|
+
}
|
|
1137
|
+
dnsCache.set(ip, {
|
|
1138
|
+
ip,
|
|
1139
|
+
trustedBot: false
|
|
1140
|
+
}).catch((err) => {
|
|
1141
|
+
this.logger.error({ err }, "Failed to save dnsCache in storage");
|
|
1142
|
+
});
|
|
1143
|
+
return false;
|
|
1144
|
+
}
|
|
1145
|
+
isBotIPTrusted(ipAddress) {
|
|
1146
|
+
return getDataSources().goodBotsDataBase(ipAddress) !== null;
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
|
|
1150
|
+
//#endregion
|
|
1151
|
+
//#region src/botDetector/checkers/goodBots/goodBots.ts
|
|
1152
|
+
const suffixPath = resolveDataPath("suffix.json");
|
|
1153
|
+
const suffixes = JSON.parse(fs.readFileSync(suffixPath, "utf-8"));
|
|
1154
|
+
const userAgents = Object.values(suffixes).flatMap((e) => Array.isArray(e.useragent) ? e.useragent : [e.useragent]).map((u) => u.toLowerCase());
|
|
1155
|
+
var GoodBotsChecker = class extends GoodBotsBase {
|
|
1156
|
+
constructor() {
|
|
1157
|
+
super(suffixes);
|
|
1158
|
+
this.name = "Good/Bad Bot Verification";
|
|
1159
|
+
this.phase = "cheap";
|
|
1160
|
+
}
|
|
1161
|
+
isEnabled(config) {
|
|
1162
|
+
return config.checkers.enableGoodBotsChecks.enable;
|
|
1163
|
+
}
|
|
1164
|
+
async run(ctx, config) {
|
|
1165
|
+
const browserType = (ctx.parsedUA.browserType ?? "").toLowerCase();
|
|
1166
|
+
const browserName = (ctx.parsedUA.browser ?? "").toLowerCase();
|
|
1167
|
+
const ipAddress = ctx.ipAddress;
|
|
1168
|
+
const score = 0;
|
|
1169
|
+
const reasons = [];
|
|
1170
|
+
const checkersConfig = config.checkers.enableGoodBotsChecks;
|
|
1171
|
+
if (!checkersConfig.enable) return {
|
|
1172
|
+
score,
|
|
1173
|
+
reasons
|
|
1174
|
+
};
|
|
1175
|
+
if (browserType !== "crawler" && browserType !== "fetcher") return {
|
|
1176
|
+
score: 0,
|
|
1177
|
+
reasons: []
|
|
1178
|
+
};
|
|
1179
|
+
const name = browserName;
|
|
1180
|
+
const botsWithoutSuffix = [
|
|
1181
|
+
"duckduckbot",
|
|
1182
|
+
"gptbot",
|
|
1183
|
+
"oai-searchbot",
|
|
1184
|
+
"chatgpt-user"
|
|
1185
|
+
].includes(name);
|
|
1186
|
+
const botsWithSuffix = userAgents.some((suf) => name.includes(suf));
|
|
1187
|
+
if (checkersConfig.banUnlistedBots && !botsWithoutSuffix && !botsWithSuffix) {
|
|
1188
|
+
reasons.push("BAD_BOT_DETECTED");
|
|
1189
|
+
return {
|
|
1190
|
+
score: 0,
|
|
1191
|
+
reasons
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
let trusted;
|
|
1195
|
+
if (botsWithSuffix) trusted = await this.isBotFromTrustedDomain(ipAddress);
|
|
1196
|
+
else trusted = this.isBotIPTrusted(ipAddress);
|
|
1197
|
+
if (!trusted) {
|
|
1198
|
+
reasons.push("BAD_BOT_DETECTED");
|
|
1199
|
+
return {
|
|
1200
|
+
score: checkersConfig.penalties,
|
|
1201
|
+
reasons
|
|
1202
|
+
};
|
|
1203
|
+
}
|
|
1204
|
+
reasons.push("GOOD_BOT_IDENTIFIED");
|
|
1205
|
+
return {
|
|
1206
|
+
score,
|
|
1207
|
+
reasons
|
|
1208
|
+
};
|
|
1209
|
+
}
|
|
1210
|
+
};
|
|
1211
|
+
CheckerRegistry.register(new GoodBotsChecker());
|
|
1212
|
+
|
|
1213
|
+
//#endregion
|
|
1214
|
+
//#region src/botDetector/checkers/browserTypesAneDevicesCalc.ts
|
|
1215
|
+
var BrowserDetailsAndDeviceChecker = class {
|
|
1216
|
+
constructor() {
|
|
1217
|
+
this.name = "Browser and Device Verification";
|
|
1218
|
+
this.phase = "cheap";
|
|
1219
|
+
}
|
|
1220
|
+
isEnabled(config) {
|
|
1221
|
+
return config.checkers.enableBrowserAndDeviceChecks.enable;
|
|
1222
|
+
}
|
|
1223
|
+
run(ctx, config) {
|
|
1224
|
+
const checkConfig = config.checkers.enableBrowserAndDeviceChecks;
|
|
1225
|
+
const reasons = [];
|
|
1226
|
+
let score = 0;
|
|
1227
|
+
if (!checkConfig.enable) return {
|
|
1228
|
+
score,
|
|
1229
|
+
reasons
|
|
1230
|
+
};
|
|
1231
|
+
const penalties = checkConfig.penalties;
|
|
1232
|
+
const bType = ctx.parsedUA.browserType;
|
|
1233
|
+
const bName = ctx.parsedUA.browser ?? "";
|
|
1234
|
+
const bOS = ctx.parsedUA.os ?? "";
|
|
1235
|
+
const dType = ctx.parsedUA.device ?? "desktop";
|
|
1236
|
+
if (bType === "cli" || bType === "library") {
|
|
1237
|
+
score += penalties.cliOrLibrary;
|
|
1238
|
+
reasons.push("CLI_OR_LIBRARY");
|
|
1239
|
+
}
|
|
1240
|
+
if ([
|
|
1241
|
+
"ie",
|
|
1242
|
+
"iemobile",
|
|
1243
|
+
"internet explorer"
|
|
1244
|
+
].includes(bName.toLowerCase())) {
|
|
1245
|
+
score += penalties.internetExplorer;
|
|
1246
|
+
reasons.push("INTERNET_EXPLORER");
|
|
1247
|
+
}
|
|
1248
|
+
if (bOS.toLowerCase().includes("kali")) {
|
|
1249
|
+
score += penalties.linuxOs;
|
|
1250
|
+
reasons.push("KALI_LINUX_OS");
|
|
1251
|
+
}
|
|
1252
|
+
if (bOS === "Mac OS" && dType === "mobile") {
|
|
1253
|
+
score += penalties.impossibleBrowserCombinations;
|
|
1254
|
+
reasons.push("IMPOSSIBLE_BROWSER_COMBINATION");
|
|
1255
|
+
}
|
|
1256
|
+
if (bName === "Safari" && bOS === "Windows") {
|
|
1257
|
+
score += penalties.impossibleBrowserCombinations;
|
|
1258
|
+
reasons.push("IMPOSSIBLE_BROWSER_COMBINATION");
|
|
1259
|
+
}
|
|
1260
|
+
if (dType === "desktop" && ctx.parsedUA.deviceVendor) {
|
|
1261
|
+
score += penalties.impossibleBrowserCombinations;
|
|
1262
|
+
reasons.push("IMPOSSIBLE_BROWSER_COMBINATION");
|
|
1263
|
+
}
|
|
1264
|
+
if (!bType && (!bName || dType !== "desktop")) {
|
|
1265
|
+
score += penalties.browserTypeUnknown;
|
|
1266
|
+
reasons.push("BROWSER_TYPE_UNKNOWN");
|
|
1267
|
+
}
|
|
1268
|
+
if (!bName) {
|
|
1269
|
+
score += penalties.browserNameUnknown;
|
|
1270
|
+
reasons.push("BROWSER_NAME_UNKNOWN");
|
|
1271
|
+
}
|
|
1272
|
+
if (dType === "desktop" && !bOS) {
|
|
1273
|
+
score += penalties.desktopWithoutOS;
|
|
1274
|
+
reasons.push("DESKTOP_WITHOUT_OS");
|
|
1275
|
+
}
|
|
1276
|
+
if (dType !== "desktop" && !ctx.parsedUA.deviceVendor) {
|
|
1277
|
+
score += penalties.deviceVendorUnknown;
|
|
1278
|
+
reasons.push("DEVICE_VENDOR_UNKNOWN");
|
|
1279
|
+
}
|
|
1280
|
+
if (!ctx.parsedUA.browserVersion) {
|
|
1281
|
+
score += penalties.browserVersionUnknown;
|
|
1282
|
+
reasons.push("BROWSER_VERSION_UNKNOWN");
|
|
1283
|
+
}
|
|
1284
|
+
if (!ctx.parsedUA.deviceModel) {
|
|
1285
|
+
score += penalties.deviceModelUnknown;
|
|
1286
|
+
reasons.push("NO_MODEL");
|
|
1287
|
+
}
|
|
1288
|
+
return {
|
|
1289
|
+
score,
|
|
1290
|
+
reasons
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
};
|
|
1294
|
+
CheckerRegistry.register(new BrowserDetailsAndDeviceChecker());
|
|
1295
|
+
|
|
1296
|
+
//#endregion
|
|
1297
|
+
//#region src/botDetector/utils/regex/acceptLangRegex.ts
|
|
1298
|
+
const space = anyOf(" ", " ").times.any();
|
|
1299
|
+
const quality = maybe(anyOf(exactly(space, ";", space, "q", space, "=", space, anyOf(exactly("0", maybe(anyOf(exactly(".", digit.times.between(1, 3))))), exactly("1", maybe(anyOf(exactly(".", exactly("0").times.between(1, 3)))))))));
|
|
1300
|
+
const primaryTag = letter.times.between(1, 8);
|
|
1301
|
+
const subTag = anyOf(exactly("-", anyOf(letter, digit).times.between(1, 8))).times.any();
|
|
1302
|
+
const language = exactly(anyOf(exactly("*"), exactly(primaryTag, subTag)), quality);
|
|
1303
|
+
const repeatingLanguages = anyOf(exactly(space, ",", space, language)).times.any();
|
|
1304
|
+
const acceptLanguageValidator = createRegExp(exactly(language, repeatingLanguages).at.lineStart().at.lineEnd(), ["i"]);
|
|
1305
|
+
|
|
1306
|
+
//#endregion
|
|
1307
|
+
//#region src/botDetector/checkers/acceptLangMap.ts
|
|
1308
|
+
var LocaleMapChecker = class {
|
|
1309
|
+
constructor() {
|
|
1310
|
+
this.name = "Locale and Country Verification";
|
|
1311
|
+
this.phase = "cheap";
|
|
1312
|
+
}
|
|
1313
|
+
isEnabled(config) {
|
|
1314
|
+
return config.checkers.localeMapsCheck.enable;
|
|
1315
|
+
}
|
|
1316
|
+
run(ctx, config) {
|
|
1317
|
+
const settings = config.checkers.localeMapsCheck;
|
|
1318
|
+
const reasons = [];
|
|
1319
|
+
let score = 0;
|
|
1320
|
+
if (!settings.enable) return {
|
|
1321
|
+
score,
|
|
1322
|
+
reasons
|
|
1323
|
+
};
|
|
1324
|
+
const AccHeader = ctx.req.get("Accept-Language") ?? "";
|
|
1325
|
+
if (!AccHeader) {
|
|
1326
|
+
score += settings.penalties.missingHeader;
|
|
1327
|
+
if (score > 0) reasons.push("LOCALE_MISMATCH");
|
|
1328
|
+
return {
|
|
1329
|
+
score,
|
|
1330
|
+
reasons
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1333
|
+
if (!acceptLanguageValidator.test(AccHeader.trim().toLowerCase())) {
|
|
1334
|
+
score += settings.penalties.malformedHeader;
|
|
1335
|
+
if (score > 0) reasons.push("LOCALE_MISMATCH");
|
|
1336
|
+
return {
|
|
1337
|
+
score,
|
|
1338
|
+
reasons
|
|
1339
|
+
};
|
|
1340
|
+
}
|
|
1341
|
+
const langs = AccHeader.split(",").map((entry) => {
|
|
1342
|
+
const [tag, q] = entry.trim().split(/\s*;\s*q\s*=\s*/);
|
|
1343
|
+
return {
|
|
1344
|
+
tag: tag.toLowerCase(),
|
|
1345
|
+
weight: q ? parseFloat(q) : 1
|
|
1346
|
+
};
|
|
1347
|
+
}).sort((a, b) => b.weight - a.weight);
|
|
1348
|
+
const country = ctx.geoData.country;
|
|
1349
|
+
const countryCode = ctx.geoData.countryCode;
|
|
1350
|
+
const iso6 = ctx.geoData.iso639;
|
|
1351
|
+
if (!country || !countryCode || !iso6) {
|
|
1352
|
+
score += settings.penalties.missingGeoData;
|
|
1353
|
+
if (score > 0) reasons.push("LOCALE_MISMATCH");
|
|
1354
|
+
return {
|
|
1355
|
+
score,
|
|
1356
|
+
reasons
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
const expectedCountryCode = countryCode.toLowerCase();
|
|
1360
|
+
const expectedLang = iso6.toLowerCase();
|
|
1361
|
+
const combined = `${expectedLang}-${expectedCountryCode}`;
|
|
1362
|
+
let localeMatchesGeo = false;
|
|
1363
|
+
for (const { tag, weight } of langs) {
|
|
1364
|
+
if (weight === 0) continue;
|
|
1365
|
+
const parts = tag.split(/[-_]/);
|
|
1366
|
+
const langPart = parts[0];
|
|
1367
|
+
const regionPart = parts.find((p) => p.length === 2 && p === expectedCountryCode);
|
|
1368
|
+
if (regionPart) {
|
|
1369
|
+
localeMatchesGeo = true;
|
|
1370
|
+
break;
|
|
1371
|
+
}
|
|
1372
|
+
if (langPart === expectedLang) {
|
|
1373
|
+
localeMatchesGeo = true;
|
|
1374
|
+
break;
|
|
1375
|
+
}
|
|
1376
|
+
if (langPart && regionPart && tag === combined) {
|
|
1377
|
+
localeMatchesGeo = true;
|
|
1378
|
+
break;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
if (!localeMatchesGeo) {
|
|
1382
|
+
score += settings.penalties.ipAndHeaderMismatch;
|
|
1383
|
+
if (score > 0) reasons.push("LOCALE_MISMATCH");
|
|
1384
|
+
}
|
|
1385
|
+
return {
|
|
1386
|
+
score,
|
|
1387
|
+
reasons
|
|
1388
|
+
};
|
|
1389
|
+
}
|
|
1390
|
+
};
|
|
1391
|
+
CheckerRegistry.register(new LocaleMapChecker());
|
|
1392
|
+
|
|
1393
|
+
//#endregion
|
|
1394
|
+
//#region src/botDetector/helpers/cache/rateLimitarCache.ts
|
|
1395
|
+
const RATE_TTL_SECONDS_FALLBACK = 120;
|
|
1396
|
+
const PREFIX$4 = "rate:";
|
|
1397
|
+
const rateCache = {
|
|
1398
|
+
async get(cookie) {
|
|
1399
|
+
return getStorage().getItem(`${PREFIX$4}${cookie}`);
|
|
1400
|
+
},
|
|
1401
|
+
async set(cookie, entry, ttlSeconds) {
|
|
1402
|
+
await getStorage().setItem(`${PREFIX$4}${cookie}`, entry, { ttl: ttlSeconds ?? RATE_TTL_SECONDS_FALLBACK });
|
|
1403
|
+
},
|
|
1404
|
+
async delete(cookie) {
|
|
1405
|
+
await getStorage().removeItem(`${PREFIX$4}${cookie}`);
|
|
1406
|
+
},
|
|
1407
|
+
async clear() {
|
|
1408
|
+
await getStorage().clear();
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
1411
|
+
|
|
1412
|
+
//#endregion
|
|
1413
|
+
//#region src/botDetector/checkers/rateTracker.ts
|
|
1414
|
+
var BehavioralDbChecker = class {
|
|
1415
|
+
constructor() {
|
|
1416
|
+
this.name = "Behavior Rate Verification";
|
|
1417
|
+
this.phase = "heavy";
|
|
1418
|
+
}
|
|
1419
|
+
get logger() {
|
|
1420
|
+
this._logger ??= getLogger().child({
|
|
1421
|
+
service: "botDetector",
|
|
1422
|
+
branch: "checker",
|
|
1423
|
+
type: "BehavioralDbChecker"
|
|
1424
|
+
});
|
|
1425
|
+
return this._logger;
|
|
1426
|
+
}
|
|
1427
|
+
isEnabled(config) {
|
|
1428
|
+
return config.checkers.enableBehaviorRateCheck.enable;
|
|
1429
|
+
}
|
|
1430
|
+
async run(ctx, config) {
|
|
1431
|
+
const cookie = ctx.cookie ?? "";
|
|
1432
|
+
const checkConfig = config.checkers.enableBehaviorRateCheck;
|
|
1433
|
+
if (!checkConfig.enable) return {
|
|
1434
|
+
score: 0,
|
|
1435
|
+
reasons: []
|
|
1436
|
+
};
|
|
1437
|
+
const BEHAVIORAL_THRESHOLD = checkConfig.behavioral_threshold;
|
|
1438
|
+
const BEHAVIORAL_WINDOW = checkConfig.behavioral_window;
|
|
1439
|
+
const BEHAVIORAL_PENALTY = checkConfig.penalties;
|
|
1440
|
+
const ttlSeconds = Math.ceil(BEHAVIORAL_WINDOW / 1e3);
|
|
1441
|
+
const cached = await rateCache.get(cookie);
|
|
1442
|
+
if (cached) if (Date.now() - cached.timestamp <= BEHAVIORAL_WINDOW) {
|
|
1443
|
+
const newCount = cached.request_count + 1;
|
|
1444
|
+
const score = newCount > BEHAVIORAL_THRESHOLD ? BEHAVIORAL_PENALTY : 0;
|
|
1445
|
+
rateCache.set(cookie, {
|
|
1446
|
+
...cached,
|
|
1447
|
+
request_count: newCount,
|
|
1448
|
+
score
|
|
1449
|
+
}, ttlSeconds).catch((err) => {
|
|
1450
|
+
this.logger.error({ err }, "Failed to save rateCache in storage");
|
|
1451
|
+
});
|
|
1452
|
+
return {
|
|
1453
|
+
score,
|
|
1454
|
+
reasons: score ? ["BEHAVIOR_TOO_FAST"] : []
|
|
1455
|
+
};
|
|
1456
|
+
} else {
|
|
1457
|
+
rateCache.set(cookie, {
|
|
1458
|
+
request_count: 1,
|
|
1459
|
+
timestamp: Date.now(),
|
|
1460
|
+
score: 0
|
|
1461
|
+
}, ttlSeconds).catch((err) => {
|
|
1462
|
+
this.logger.error({ err }, "Failed to reset rateCache in storage");
|
|
1463
|
+
});
|
|
1464
|
+
return {
|
|
1465
|
+
score: 0,
|
|
1466
|
+
reasons: []
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
return {
|
|
1470
|
+
score: 0,
|
|
1471
|
+
reasons: []
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
};
|
|
1475
|
+
CheckerRegistry.register(new BehavioralDbChecker());
|
|
1476
|
+
|
|
1477
|
+
//#endregion
|
|
1478
|
+
//#region src/botDetector/checkers/proxyISPAndCookieCalc.ts
|
|
1479
|
+
var ProxyIspAndCookieChecker = class {
|
|
1480
|
+
constructor() {
|
|
1481
|
+
this.name = "Proxy, ISP and Cookie Verification";
|
|
1482
|
+
this.phase = "heavy";
|
|
1483
|
+
}
|
|
1484
|
+
isEnabled(config) {
|
|
1485
|
+
return config.checkers.enableProxyIspCookiesChecks.enable;
|
|
1486
|
+
}
|
|
1487
|
+
run(ctx, config) {
|
|
1488
|
+
const checkConfig = config.checkers.enableProxyIspCookiesChecks;
|
|
1489
|
+
const reasons = [];
|
|
1490
|
+
let score = 0;
|
|
1491
|
+
if (!checkConfig.enable) return {
|
|
1492
|
+
score,
|
|
1493
|
+
reasons
|
|
1494
|
+
};
|
|
1495
|
+
const { penalties } = checkConfig;
|
|
1496
|
+
const cookie = ctx.cookie ?? "";
|
|
1497
|
+
const proxy = ctx.proxy.isProxy;
|
|
1498
|
+
const proxyType = ctx.proxy.proxyType ?? "";
|
|
1499
|
+
const hosting = ctx.geoData.hosting ?? false;
|
|
1500
|
+
const isp = ctx.geoData.isp ?? "";
|
|
1501
|
+
const org = ctx.geoData.org ?? "";
|
|
1502
|
+
if (!cookie) {
|
|
1503
|
+
score += penalties.cookieMissing;
|
|
1504
|
+
reasons.push("COOKIE_MISSING");
|
|
1505
|
+
}
|
|
1506
|
+
if (proxy) {
|
|
1507
|
+
score += penalties.proxyDetected;
|
|
1508
|
+
reasons.push("PROXY_DETECTED");
|
|
1509
|
+
if (proxyType) {
|
|
1510
|
+
const sourceCount = proxyType.split(",").length;
|
|
1511
|
+
if (sourceCount >= 4) score += penalties.multiSourceBonus4plus;
|
|
1512
|
+
else if (sourceCount >= 2) score += penalties.multiSourceBonus2to3;
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
if (hosting) {
|
|
1516
|
+
score += penalties.hostingDetected;
|
|
1517
|
+
reasons.push("HOSTING_DETECTED");
|
|
1518
|
+
}
|
|
1519
|
+
if (!isp) {
|
|
1520
|
+
score += penalties.ispUnknown;
|
|
1521
|
+
reasons.push("ISP_UNKNOWN");
|
|
1522
|
+
}
|
|
1523
|
+
if (!org) {
|
|
1524
|
+
score += penalties.orgUnknown;
|
|
1525
|
+
reasons.push("ORG_UNKNOWN");
|
|
1526
|
+
}
|
|
1527
|
+
return {
|
|
1528
|
+
score,
|
|
1529
|
+
reasons
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
};
|
|
1533
|
+
CheckerRegistry.register(new ProxyIspAndCookieChecker());
|
|
1534
|
+
|
|
1535
|
+
//#endregion
|
|
1536
|
+
//#region src/botDetector/checkers/headers/headersBase.ts
|
|
1537
|
+
var HeadersBase = class {
|
|
1538
|
+
constructor() {
|
|
1539
|
+
this.config = getConfiguration().headerOptions;
|
|
1540
|
+
}
|
|
1541
|
+
mustHaveHeadersChecker(req) {
|
|
1542
|
+
let score = 0;
|
|
1543
|
+
if (req.httpVersion === "1.0") return score += 40;
|
|
1544
|
+
if (!req.get("User-Agent")) score += this.config.weightPerMustHeader;
|
|
1545
|
+
if (!req.accepts) score += this.config.weightPerMustHeader;
|
|
1546
|
+
if (!req.acceptsEncodings) score += this.config.weightPerMustHeader;
|
|
1547
|
+
if (!req.acceptsLanguages) score += this.config.weightPerMustHeader;
|
|
1548
|
+
if (!req.host) score += this.config.weightPerMustHeader;
|
|
1549
|
+
if (!req.get("Upgrade-Insecure-Requests")) score += this.config.weightPerMustHeader;
|
|
1550
|
+
if (!req.get("x-client-id")) score += this.config.weightPerMustHeader;
|
|
1551
|
+
if (req.httpVersion === "1.1" || req.get("Connection")) {
|
|
1552
|
+
if (!req.get("Connection") || req.get("Connection") !== "keep-alive") score += this.config.connectionHeaderIsClose;
|
|
1553
|
+
}
|
|
1554
|
+
if (!req.get("sec-fetch-mode") || !req.get("sec-fetch-dest") || !req.get("sec-fetch-site")) score += this.config.weightPerMustHeader;
|
|
1555
|
+
return score;
|
|
1556
|
+
}
|
|
1557
|
+
async engineHeaders(req) {
|
|
1558
|
+
let score = 0;
|
|
1559
|
+
const ua = req.get("User-Agent");
|
|
1560
|
+
const hints = req.headers;
|
|
1561
|
+
const { name } = await new UAParser(ua, hints).getEngine().withClientHints();
|
|
1562
|
+
if (!name) return score += this.config.missingBrowserEngine;
|
|
1563
|
+
const headers = Object.keys(hints);
|
|
1564
|
+
const containsCh = headers.some((sec) => sec.toLowerCase().startsWith("sec-ch-ua"));
|
|
1565
|
+
const containsTe = headers.some((te) => te.toLowerCase() === "te");
|
|
1566
|
+
if (name === "Blink") {
|
|
1567
|
+
if (!containsCh) score += this.config.clientHintsMissingForBlink;
|
|
1568
|
+
if (containsTe) score += this.config.teHeaderUnexpectedForBlink;
|
|
1569
|
+
} else if (name === "Gecko") {
|
|
1570
|
+
if (containsCh) score += this.config.clientHintsUnexpectedForGecko;
|
|
1571
|
+
if (!containsTe) score += this.config.teHeaderMissingForGecko;
|
|
1572
|
+
} else if (name === "WebKit") {
|
|
1573
|
+
if (containsCh) score += this.config.clientHintsUnexpectedForGecko;
|
|
1574
|
+
if (containsTe) score += this.config.teHeaderUnexpectedForBlink;
|
|
1575
|
+
}
|
|
1576
|
+
return score;
|
|
1577
|
+
}
|
|
1578
|
+
weirdHeaders(req) {
|
|
1579
|
+
let score = 0;
|
|
1580
|
+
if (req.get("accept") === "*/*") score += this.config.omittedAcceptHeader;
|
|
1581
|
+
if (req.get("x-requested-with") && req.method === "GET") score += this.config.AJAXHeaderExists;
|
|
1582
|
+
if (req.get("postman-token") || req.get("insomnia")) score += this.config.postManOrInsomiaHeaders;
|
|
1583
|
+
const hostHeader = req.get("X-Forwarded-Host");
|
|
1584
|
+
if (hostHeader && req.hostname && hostHeader !== req.hostname) score += this.config.hostMismatchWeight;
|
|
1585
|
+
if (req.method === "GET" && req.get("Cache-Control") === "no-cache" && req.get("Pragma") === "no-cache") score += this.config.aggressiveCacheControlOnGet;
|
|
1586
|
+
if (req.get("sec-fetch-site") === "cross-site" && !req.get("referer")) score += this.config.crossSiteRequestMissingReferer;
|
|
1587
|
+
const isBrowserRequest = !req.get("x-client-id");
|
|
1588
|
+
const isTopNavigation = req.method === "GET" && req.get("sec-fetch-dest") === "document";
|
|
1589
|
+
if (isBrowserRequest && !isTopNavigation && req.method !== "GET") {
|
|
1590
|
+
const origin = req.get("origin");
|
|
1591
|
+
if (!origin) score += this.config.originHeaderIsNULL;
|
|
1592
|
+
else if (origin !== `${req.protocol}://${req.hostname}`) score += this.config.originHeaderMismatch;
|
|
1593
|
+
}
|
|
1594
|
+
const mode = req.get("sec-fetch-mode");
|
|
1595
|
+
if (mode !== "same-origin" && mode !== "navigate") score += this.config.inconsistentSecFetchMode;
|
|
1596
|
+
return score;
|
|
1597
|
+
}
|
|
1598
|
+
};
|
|
1599
|
+
|
|
1600
|
+
//#endregion
|
|
1601
|
+
//#region src/botDetector/checkers/headers/headers.ts
|
|
1602
|
+
var HeaderAnalysis = class extends HeadersBase {
|
|
1603
|
+
constructor(req) {
|
|
1604
|
+
super();
|
|
1605
|
+
this.req = req;
|
|
1606
|
+
}
|
|
1607
|
+
async scoreHeaders() {
|
|
1608
|
+
const missing = this.mustHaveHeadersChecker(this.req);
|
|
1609
|
+
const engines = await this.engineHeaders(this.req);
|
|
1610
|
+
const weird = this.weirdHeaders(this.req);
|
|
1611
|
+
return missing + engines + weird;
|
|
1612
|
+
}
|
|
1613
|
+
};
|
|
1614
|
+
|
|
1615
|
+
//#endregion
|
|
1616
|
+
//#region src/botDetector/utils/regex/pathTravelersRegex.ts
|
|
1617
|
+
const startOrSlash = anyOf(exactly("").at.lineStart(), "/").grouped();
|
|
1618
|
+
const slashOrEnd = anyOf("/", exactly("").at.lineEnd());
|
|
1619
|
+
const pathRules = [
|
|
1620
|
+
{
|
|
1621
|
+
re: createRegExp(startOrSlash, ".git", slashOrEnd, ["i"]),
|
|
1622
|
+
weight: 10
|
|
1623
|
+
},
|
|
1624
|
+
{
|
|
1625
|
+
re: createRegExp(startOrSlash, ".git/config", slashOrEnd, ["i"]),
|
|
1626
|
+
weight: 10
|
|
1627
|
+
},
|
|
1628
|
+
{
|
|
1629
|
+
re: createRegExp(startOrSlash, ".env", maybe(anyOf(".local", ".example")), slashOrEnd, ["i"]),
|
|
1630
|
+
weight: 10
|
|
1631
|
+
},
|
|
1632
|
+
{
|
|
1633
|
+
re: createRegExp(startOrSlash, "wp-admin", slashOrEnd, ["i"]),
|
|
1634
|
+
weight: 8
|
|
1635
|
+
},
|
|
1636
|
+
{
|
|
1637
|
+
re: createRegExp("/wp-json/wp/v2", slashOrEnd, ["i"]),
|
|
1638
|
+
weight: 7
|
|
1639
|
+
},
|
|
1640
|
+
{
|
|
1641
|
+
re: createRegExp("/", anyOf("jenkins", "hudson").grouped(), slashOrEnd, ["i"]),
|
|
1642
|
+
weight: 9
|
|
1643
|
+
},
|
|
1644
|
+
{
|
|
1645
|
+
re: createRegExp("/", anyOf("script", "login").grouped(), ".groovy", slashOrEnd, ["i"]),
|
|
1646
|
+
weight: 8
|
|
1647
|
+
},
|
|
1648
|
+
{
|
|
1649
|
+
re: createRegExp("/actuator/", anyOf("env", "health", "metrics"), slashOrEnd, ["i"]),
|
|
1650
|
+
weight: 8
|
|
1651
|
+
},
|
|
1652
|
+
{
|
|
1653
|
+
re: createRegExp("/web.config", slashOrEnd, ["i"]),
|
|
1654
|
+
weight: 7
|
|
1655
|
+
},
|
|
1656
|
+
{
|
|
1657
|
+
re: createRegExp("/.DS_Store", slashOrEnd, ["i"]),
|
|
1658
|
+
weight: 4
|
|
1659
|
+
},
|
|
1660
|
+
{
|
|
1661
|
+
re: createRegExp("/latest/meta-data/iam", slashOrEnd, ["i"]),
|
|
1662
|
+
weight: 10
|
|
1663
|
+
},
|
|
1664
|
+
{
|
|
1665
|
+
re: createRegExp(startOrSlash, ".aws/credentials", slashOrEnd, ["i"]),
|
|
1666
|
+
weight: 10
|
|
1667
|
+
},
|
|
1668
|
+
{
|
|
1669
|
+
re: createRegExp("/Dockerfile", slashOrEnd, ["i"]),
|
|
1670
|
+
weight: 7
|
|
1671
|
+
},
|
|
1672
|
+
{
|
|
1673
|
+
re: createRegExp("/composer.lock", slashOrEnd, ["i"]),
|
|
1674
|
+
weight: 7
|
|
1675
|
+
},
|
|
1676
|
+
{
|
|
1677
|
+
re: createRegExp("/jnlpJars/jenkins-cli.jar", slashOrEnd, ["i"]),
|
|
1678
|
+
weight: 9
|
|
1679
|
+
},
|
|
1680
|
+
{
|
|
1681
|
+
re: createRegExp("/manager/html", slashOrEnd, ["i"]),
|
|
1682
|
+
weight: 10
|
|
1683
|
+
},
|
|
1684
|
+
{
|
|
1685
|
+
re: createRegExp("/shell", maybe(".php"), slashOrEnd, ["i"]),
|
|
1686
|
+
weight: 10
|
|
1687
|
+
},
|
|
1688
|
+
{
|
|
1689
|
+
re: createRegExp("/", anyOf("grafana", "kibana", "prometheus").grouped(), slashOrEnd, ["i"]),
|
|
1690
|
+
weight: 7
|
|
1691
|
+
},
|
|
1692
|
+
{
|
|
1693
|
+
re: createRegExp("/", anyOf("owa", "ecp").grouped(), slashOrEnd, ["i"]),
|
|
1694
|
+
weight: 8
|
|
1695
|
+
},
|
|
1696
|
+
{
|
|
1697
|
+
re: createRegExp("wp-login.php", slashOrEnd, ["i"]),
|
|
1698
|
+
weight: 8
|
|
1699
|
+
},
|
|
1700
|
+
{
|
|
1701
|
+
re: createRegExp("/swagger-ui", anyOf("/", ".html"), slashOrEnd, ["i"]),
|
|
1702
|
+
weight: 6
|
|
1703
|
+
},
|
|
1704
|
+
{
|
|
1705
|
+
re: createRegExp("/v2/api-docs", slashOrEnd, ["i"]),
|
|
1706
|
+
weight: 5
|
|
1707
|
+
},
|
|
1708
|
+
{
|
|
1709
|
+
re: createRegExp("/api-docs", anyOf("/", ".json"), slashOrEnd, ["i"]),
|
|
1710
|
+
weight: 5
|
|
1711
|
+
},
|
|
1712
|
+
{
|
|
1713
|
+
re: createRegExp("phpmyadmin", slashOrEnd, ["i"]),
|
|
1714
|
+
weight: 8
|
|
1715
|
+
},
|
|
1716
|
+
{
|
|
1717
|
+
re: createRegExp(startOrSlash, ".well-known", slashOrEnd, ["i"]),
|
|
1718
|
+
weight: 5
|
|
1719
|
+
},
|
|
1720
|
+
{
|
|
1721
|
+
re: createRegExp(startOrSlash, ".htaccess", slashOrEnd, ["i"]),
|
|
1722
|
+
weight: 7
|
|
1723
|
+
},
|
|
1724
|
+
{
|
|
1725
|
+
re: createRegExp("composer.json", slashOrEnd, ["i"]),
|
|
1726
|
+
weight: 7
|
|
1727
|
+
},
|
|
1728
|
+
{
|
|
1729
|
+
re: createRegExp("docker-compose.y", maybe("a"), "ml", slashOrEnd, ["i"]),
|
|
1730
|
+
weight: 7
|
|
1731
|
+
},
|
|
1732
|
+
{
|
|
1733
|
+
re: createRegExp(".", anyOf("sql", "bak", "old", "save", "log", "ini", "conf", "zip", exactly("tar", maybe(".gz"))), slashOrEnd, ["i"]),
|
|
1734
|
+
weight: 7
|
|
1735
|
+
},
|
|
1736
|
+
{
|
|
1737
|
+
re: createRegExp(anyOf("..", "%2e%2e").grouped(), anyOf("/", "\\"), ["i"]),
|
|
1738
|
+
weight: 5
|
|
1739
|
+
},
|
|
1740
|
+
{
|
|
1741
|
+
re: createRegExp(anyOf("package-lock.json", "yarn.lock", ".gitignore").grouped(), slashOrEnd, ["i"]),
|
|
1742
|
+
weight: 6
|
|
1743
|
+
},
|
|
1744
|
+
{
|
|
1745
|
+
re: createRegExp(startOrSlash, "admin", slashOrEnd, ["i"]),
|
|
1746
|
+
weight: 6
|
|
1747
|
+
},
|
|
1748
|
+
{
|
|
1749
|
+
re: createRegExp("phpinfo", maybe(".php"), slashOrEnd, ["i"]),
|
|
1750
|
+
weight: 8
|
|
1751
|
+
},
|
|
1752
|
+
{
|
|
1753
|
+
re: createRegExp(startOrSlash, ".ssh", slashOrEnd, ["i"]),
|
|
1754
|
+
weight: 8
|
|
1755
|
+
},
|
|
1756
|
+
{
|
|
1757
|
+
re: createRegExp(exactly("").at.lineStart(), "/xmlrpc.php", slashOrEnd, ["i"]),
|
|
1758
|
+
weight: 8
|
|
1759
|
+
},
|
|
1760
|
+
{
|
|
1761
|
+
re: createRegExp(startOrSlash, "wp-config.php", slashOrEnd, ["i"]),
|
|
1762
|
+
weight: 10
|
|
1763
|
+
},
|
|
1764
|
+
{
|
|
1765
|
+
re: createRegExp(startOrSlash, anyOf("install", "setup").grouped(), maybe(".php"), slashOrEnd, ["i"]),
|
|
1766
|
+
weight: 8
|
|
1767
|
+
},
|
|
1768
|
+
{
|
|
1769
|
+
re: createRegExp(startOrSlash, "backup", maybe("s"), anyOf(".zip", ".tar.gz", ".sql"), slashOrEnd, ["i"]),
|
|
1770
|
+
weight: 8
|
|
1771
|
+
},
|
|
1772
|
+
{
|
|
1773
|
+
re: createRegExp(startOrSlash, ".gitlab-ci.yml", slashOrEnd, ["i"]),
|
|
1774
|
+
weight: 8
|
|
1775
|
+
},
|
|
1776
|
+
{
|
|
1777
|
+
re: createRegExp(startOrSlash, ".svn", slashOrEnd, ["i"]),
|
|
1778
|
+
weight: 6
|
|
1779
|
+
},
|
|
1780
|
+
{
|
|
1781
|
+
re: createRegExp(startOrSlash, ".hg", slashOrEnd, ["i"]),
|
|
1782
|
+
weight: 6
|
|
1783
|
+
},
|
|
1784
|
+
{
|
|
1785
|
+
re: createRegExp(startOrSlash, "CVS", slashOrEnd, ["i"]),
|
|
1786
|
+
weight: 6
|
|
1787
|
+
},
|
|
1788
|
+
{
|
|
1789
|
+
re: createRegExp(startOrSlash, ".vscode", slashOrEnd, ["i"]),
|
|
1790
|
+
weight: 4
|
|
1791
|
+
},
|
|
1792
|
+
{
|
|
1793
|
+
re: createRegExp(startOrSlash, ".htpasswd", slashOrEnd, ["i"]),
|
|
1794
|
+
weight: 8
|
|
1795
|
+
},
|
|
1796
|
+
{
|
|
1797
|
+
re: createRegExp("phpunit", maybe(".phar"), slashOrEnd, ["i"]),
|
|
1798
|
+
weight: 9
|
|
1799
|
+
},
|
|
1800
|
+
{
|
|
1801
|
+
re: createRegExp(startOrSlash, "config", maybe("uration"), ".", anyOf("php", "yml", "json", "xml", "ini", "conf", "cfg"), slashOrEnd, ["i"]),
|
|
1802
|
+
weight: 7
|
|
1803
|
+
}
|
|
1804
|
+
];
|
|
1805
|
+
const whiteList = [
|
|
1806
|
+
createRegExp(exactly("").at.lineStart(), maybe("/"), exactly("").at.lineEnd()),
|
|
1807
|
+
createRegExp(exactly("").at.lineStart(), "/", anyOf("css", "js", "images", "assets", "static").grouped(), "/", ["i"]),
|
|
1808
|
+
createRegExp(".", anyOf("html", "css", "js", "png", exactly("jp", maybe("e"), "g"), "svg", "map", exactly("woff", maybe("2")), "ttf", "webp"), exactly("").at.lineEnd(), ["i"]),
|
|
1809
|
+
createRegExp(exactly("").at.lineStart(), "/robots.txt", exactly("").at.lineEnd(), ["i"]),
|
|
1810
|
+
createRegExp(exactly("").at.lineStart(), "/sitemap.xml", exactly("").at.lineEnd(), ["i"])
|
|
1811
|
+
];
|
|
1812
|
+
|
|
1813
|
+
//#endregion
|
|
1814
|
+
//#region src/botDetector/checkers/UaAndHeaderChecker/base.ts
|
|
1815
|
+
var UaAndHeaderCheckerBase = class {
|
|
1816
|
+
pathScore(req, config) {
|
|
1817
|
+
const settings = config.pathTraveler;
|
|
1818
|
+
const MAX_DECODE_ITERATIONS = settings.maxIterations;
|
|
1819
|
+
const MAX_PATH_LENGTH = settings.maxPathLength;
|
|
1820
|
+
let score = 0;
|
|
1821
|
+
const hostHeader = req.get("x-forwarded-host");
|
|
1822
|
+
const base = `${req.protocol}://${hostHeader ?? req.get("host") ?? ""}`;
|
|
1823
|
+
const rawPath = req.get("x-original-path") || req.originalUrl;
|
|
1824
|
+
if (/(?:\.\.[\\/]|\.\.%2f|\.\.%5c|%2e%2e[\\/]|%2e%2e%2f|%2e%2e%5c)/i.test(rawPath)) return settings.traversalDetected;
|
|
1825
|
+
let pathname;
|
|
1826
|
+
try {
|
|
1827
|
+
pathname = new URL$1(rawPath, base).pathname;
|
|
1828
|
+
} catch {
|
|
1829
|
+
pathname = rawPath.split("?")[0];
|
|
1830
|
+
}
|
|
1831
|
+
if (pathname.length > MAX_PATH_LENGTH) return settings.pathLengthToLong;
|
|
1832
|
+
if (whiteList.some((rx) => rx.test(pathname))) return 0;
|
|
1833
|
+
let decoded = pathname;
|
|
1834
|
+
let totalDecodedLength = 0;
|
|
1835
|
+
for (let i = 0; i < MAX_DECODE_ITERATIONS; i++) try {
|
|
1836
|
+
const tmp = decodeURIComponent(decoded);
|
|
1837
|
+
if (tmp === decoded) break;
|
|
1838
|
+
totalDecodedLength += tmp.length;
|
|
1839
|
+
if (totalDecodedLength > MAX_PATH_LENGTH * 2) return settings.longDecoding;
|
|
1840
|
+
decoded = tmp;
|
|
1841
|
+
} catch {
|
|
1842
|
+
break;
|
|
1843
|
+
}
|
|
1844
|
+
const normalized = path$1.normalize(decoded);
|
|
1845
|
+
for (const { re, weight } of pathRules) if (re.test(normalized)) {
|
|
1846
|
+
score += weight;
|
|
1847
|
+
if (score >= 30) break;
|
|
1848
|
+
}
|
|
1849
|
+
return score;
|
|
1850
|
+
}
|
|
1851
|
+
tlsBotScore(req, config) {
|
|
1852
|
+
const settings = config.checkers.enableUaAndHeaderChecks;
|
|
1853
|
+
if (!settings.enable) return 0;
|
|
1854
|
+
const { penalties } = settings;
|
|
1855
|
+
let score = 0;
|
|
1856
|
+
const proto = req.httpVersion === "2.0" ? "h2" : req.httpVersion === "1.1" ? "http/1.1" : "";
|
|
1857
|
+
if (!proto || proto !== "h2" && proto !== "http/1.1") score += penalties.tlsCheckFailed;
|
|
1858
|
+
const cipher = req.get("x-client-cipher") ?? "";
|
|
1859
|
+
if (!new Set([
|
|
1860
|
+
"TLS_AES_128_GCM_SHA256",
|
|
1861
|
+
"TLS_AES_256_GCM_SHA384",
|
|
1862
|
+
"TLS_CHACHA20_POLY1305_SHA256",
|
|
1863
|
+
"ECDHE-ECDSA-AES128-GCM-SHA256",
|
|
1864
|
+
"ECDHE-RSA-AES128-GCM-SHA256",
|
|
1865
|
+
"ECDHE-ECDSA-CHACHA20-POLY1305",
|
|
1866
|
+
"ECDHE-RSA-CHACHA20-POLY1305",
|
|
1867
|
+
"ECDHE-ECDSA-AES256-GCM-SHA384",
|
|
1868
|
+
"ECDHE-RSA-AES256-GCM-SHA384",
|
|
1869
|
+
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
|
|
1870
|
+
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
|
|
1871
|
+
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
|
|
1872
|
+
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
|
|
1873
|
+
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
|
|
1874
|
+
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"
|
|
1875
|
+
]).has(cipher)) score += penalties.tlsCheckFailed;
|
|
1876
|
+
const tlsVersion = (req.get("x-client-tls-version") ?? "").toLowerCase();
|
|
1877
|
+
if (tlsVersion && !tlsVersion.startsWith("tls1.3") && !tlsVersion.startsWith("tls1.2")) score += penalties.tlsCheckFailed;
|
|
1878
|
+
return score;
|
|
1879
|
+
}
|
|
1880
|
+
};
|
|
1881
|
+
|
|
1882
|
+
//#endregion
|
|
1883
|
+
//#region src/botDetector/checkers/UaAndHeaderChecker/headersAndUACalc.ts
|
|
1884
|
+
var UaAndHeaderChecker = class extends UaAndHeaderCheckerBase {
|
|
1885
|
+
constructor(..._args) {
|
|
1886
|
+
super(..._args);
|
|
1887
|
+
this.name = "User agent and Header Verification";
|
|
1888
|
+
this.phase = "heavy";
|
|
1889
|
+
}
|
|
1890
|
+
isEnabled(config) {
|
|
1891
|
+
return config.checkers.enableUaAndHeaderChecks.enable;
|
|
1892
|
+
}
|
|
1893
|
+
async run(ctx, config) {
|
|
1894
|
+
const checkConfig = config.checkers.enableUaAndHeaderChecks;
|
|
1895
|
+
const reasons = [];
|
|
1896
|
+
let score = 0;
|
|
1897
|
+
if (!checkConfig.enable) return {
|
|
1898
|
+
score,
|
|
1899
|
+
reasons
|
|
1900
|
+
};
|
|
1901
|
+
const { penalties } = checkConfig;
|
|
1902
|
+
const req = ctx.req;
|
|
1903
|
+
const uaString = req.get("User-Agent") ?? "";
|
|
1904
|
+
const uaLower = uaString.toLowerCase();
|
|
1905
|
+
if (createRegExp(anyOf("headless", "puppeteer", "selenium", "playwright", "phantomjs")).test(uaLower)) {
|
|
1906
|
+
score += penalties.headlessBrowser;
|
|
1907
|
+
reasons.push("HEADLESS_BROWSER_DETECTED");
|
|
1908
|
+
}
|
|
1909
|
+
if (!uaString || uaString.length < 10) {
|
|
1910
|
+
score += penalties.shortUserAgent;
|
|
1911
|
+
reasons.push("SHORT_USER_AGENT");
|
|
1912
|
+
}
|
|
1913
|
+
const tlsCheckScore = this.tlsBotScore(req, config);
|
|
1914
|
+
if (tlsCheckScore > 0) {
|
|
1915
|
+
score += tlsCheckScore;
|
|
1916
|
+
reasons.push("TLS_CHECK_FAILED");
|
|
1917
|
+
}
|
|
1918
|
+
const headerChecker = await new HeaderAnalysis(req).scoreHeaders();
|
|
1919
|
+
if (headerChecker > 0) {
|
|
1920
|
+
score += headerChecker;
|
|
1921
|
+
reasons.push("HEADER_SCORE_TOO_HIGH");
|
|
1922
|
+
}
|
|
1923
|
+
const pathChecker = this.pathScore(req, config);
|
|
1924
|
+
if (pathChecker > 0) {
|
|
1925
|
+
score += pathChecker;
|
|
1926
|
+
reasons.push("PATH_TRAVELER_FOUND");
|
|
1927
|
+
}
|
|
1928
|
+
return {
|
|
1929
|
+
score,
|
|
1930
|
+
reasons
|
|
1931
|
+
};
|
|
1932
|
+
}
|
|
1933
|
+
};
|
|
1934
|
+
CheckerRegistry.register(new UaAndHeaderChecker());
|
|
1935
|
+
|
|
1936
|
+
//#endregion
|
|
1937
|
+
//#region src/botDetector/checkers/geoLocationCalc.ts
|
|
1938
|
+
var GeoLocationChecker = class {
|
|
1939
|
+
constructor() {
|
|
1940
|
+
this.name = "Geo-Location Verification";
|
|
1941
|
+
this.phase = "heavy";
|
|
1942
|
+
}
|
|
1943
|
+
isEnabled(config) {
|
|
1944
|
+
return config.checkers.enableGeoChecks.enable;
|
|
1945
|
+
}
|
|
1946
|
+
isAllowedCountry(country, bannedCountries) {
|
|
1947
|
+
return !bannedCountries.includes(country.trim().toLowerCase());
|
|
1948
|
+
}
|
|
1949
|
+
run(ctx, config) {
|
|
1950
|
+
const checkConfig = config.checkers.enableGeoChecks;
|
|
1951
|
+
const reasons = [];
|
|
1952
|
+
let score = 0;
|
|
1953
|
+
if (!checkConfig.enable) return {
|
|
1954
|
+
score,
|
|
1955
|
+
reasons
|
|
1956
|
+
};
|
|
1957
|
+
const penalties = checkConfig.penalties;
|
|
1958
|
+
const banScore = config.banScore;
|
|
1959
|
+
const details = ctx.geoData;
|
|
1960
|
+
const country = details.country;
|
|
1961
|
+
const countryCode = details.countryCode;
|
|
1962
|
+
if (countryCode || country) {
|
|
1963
|
+
const banned = checkConfig.bannedCountries;
|
|
1964
|
+
const codeMatch = !!countryCode && !this.isAllowedCountry(countryCode, banned);
|
|
1965
|
+
const nameMatch = !!country && !this.isAllowedCountry(country, banned);
|
|
1966
|
+
if (codeMatch || nameMatch) {
|
|
1967
|
+
score += banScore;
|
|
1968
|
+
reasons.push("BANNED_COUNTRY");
|
|
1969
|
+
}
|
|
1970
|
+
} else {
|
|
1971
|
+
score += penalties.countryUnknown;
|
|
1972
|
+
reasons.push("COUNTRY_UNKNOWN");
|
|
1973
|
+
}
|
|
1974
|
+
const region = details.region;
|
|
1975
|
+
const regionName = details.regionName;
|
|
1976
|
+
if (!region || !regionName) {
|
|
1977
|
+
score += penalties.regionUnknown;
|
|
1978
|
+
reasons.push("REGION_UNKNOWN");
|
|
1979
|
+
}
|
|
1980
|
+
if (!details.lat || !details.lon) {
|
|
1981
|
+
score += penalties.latLonUnknown;
|
|
1982
|
+
reasons.push("LAT_LON_UNKNOWN");
|
|
1983
|
+
}
|
|
1984
|
+
if (!details.district) {
|
|
1985
|
+
score += penalties.districtUnknown;
|
|
1986
|
+
reasons.push("DISTRICT_UNKNOWN");
|
|
1987
|
+
}
|
|
1988
|
+
if (!details.city) {
|
|
1989
|
+
score += penalties.cityUnknown;
|
|
1990
|
+
reasons.push("CITY_UNKNOWN");
|
|
1991
|
+
}
|
|
1992
|
+
if (!details.timezone) {
|
|
1993
|
+
score += penalties.timezoneUnknown;
|
|
1994
|
+
reasons.push("TIMEZONE_UNKNOWN");
|
|
1995
|
+
}
|
|
1996
|
+
if (!details.subregion) {
|
|
1997
|
+
score += penalties.subregionUnknown;
|
|
1998
|
+
reasons.push("SUBREGION_UNKNOWN");
|
|
1999
|
+
}
|
|
2000
|
+
if (!details.phone) {
|
|
2001
|
+
score += penalties.phoneUnknown;
|
|
2002
|
+
reasons.push("PHONE_UNKNOWN");
|
|
2003
|
+
}
|
|
2004
|
+
if (!details.continent) {
|
|
2005
|
+
score += penalties.continentUnknown;
|
|
2006
|
+
reasons.push("CONTINENT_UNKNOWN");
|
|
2007
|
+
}
|
|
2008
|
+
return {
|
|
2009
|
+
score,
|
|
2010
|
+
reasons
|
|
2011
|
+
};
|
|
2012
|
+
}
|
|
2013
|
+
};
|
|
2014
|
+
CheckerRegistry.register(new GeoLocationChecker());
|
|
2015
|
+
|
|
2016
|
+
//#endregion
|
|
2017
|
+
//#region src/botDetector/checkers/fireholEscalation.ts
|
|
2018
|
+
var ThreatLevels = class {
|
|
2019
|
+
constructor() {
|
|
2020
|
+
this.name = "Known ThreatLevels";
|
|
2021
|
+
this.phase = "cheap";
|
|
2022
|
+
}
|
|
2023
|
+
isEnabled(config) {
|
|
2024
|
+
return config.checkers.enableKnownThreatsDetections.enable;
|
|
2025
|
+
}
|
|
2026
|
+
run(ctx, config) {
|
|
2027
|
+
const { enableKnownThreatsDetections } = config.checkers;
|
|
2028
|
+
const reasons = [];
|
|
2029
|
+
let score = 0;
|
|
2030
|
+
if (!enableKnownThreatsDetections.enable) return {
|
|
2031
|
+
score,
|
|
2032
|
+
reasons
|
|
2033
|
+
};
|
|
2034
|
+
const { anonymiseNetwork, threatLevels } = enableKnownThreatsDetections.penalties;
|
|
2035
|
+
if (ctx.anon) {
|
|
2036
|
+
score += anonymiseNetwork;
|
|
2037
|
+
reasons.push("ANONYMITY_NETWORK");
|
|
2038
|
+
}
|
|
2039
|
+
switch (ctx.threatLevel) {
|
|
2040
|
+
case 1:
|
|
2041
|
+
score += threatLevels.criticalLevel1;
|
|
2042
|
+
reasons.push("FIREHOL_L1_THREAT");
|
|
2043
|
+
break;
|
|
2044
|
+
case 2:
|
|
2045
|
+
score += threatLevels.currentAttacksLevel2;
|
|
2046
|
+
reasons.push("FIREHOL_L2_THREAT");
|
|
2047
|
+
break;
|
|
2048
|
+
case 3:
|
|
2049
|
+
score += threatLevels.threatLevel3;
|
|
2050
|
+
reasons.push("FIREHOL_L3_THREAT");
|
|
2051
|
+
break;
|
|
2052
|
+
case 4:
|
|
2053
|
+
score += threatLevels.threatLevel4;
|
|
2054
|
+
reasons.push("FIREHOL_L4_THREAT");
|
|
2055
|
+
break;
|
|
2056
|
+
}
|
|
2057
|
+
return {
|
|
2058
|
+
score,
|
|
2059
|
+
reasons
|
|
2060
|
+
};
|
|
2061
|
+
}
|
|
2062
|
+
};
|
|
2063
|
+
CheckerRegistry.register(new ThreatLevels());
|
|
2064
|
+
|
|
2065
|
+
//#endregion
|
|
2066
|
+
//#region src/botDetector/checkers/asnClassification.ts
|
|
2067
|
+
var AsnClassificationChecker = class {
|
|
2068
|
+
constructor() {
|
|
2069
|
+
this.name = "ASN Classification";
|
|
2070
|
+
this.phase = "cheap";
|
|
2071
|
+
}
|
|
2072
|
+
isEnabled(config) {
|
|
2073
|
+
return config.checkers.enableAsnClassification.enable;
|
|
2074
|
+
}
|
|
2075
|
+
run(ctx, config) {
|
|
2076
|
+
const checkConfig = config.checkers.enableAsnClassification;
|
|
2077
|
+
const reasons = [];
|
|
2078
|
+
let score = 0;
|
|
2079
|
+
if (!checkConfig.enable) return {
|
|
2080
|
+
score,
|
|
2081
|
+
reasons
|
|
2082
|
+
};
|
|
2083
|
+
const { penalties } = checkConfig;
|
|
2084
|
+
const { classification, hits } = ctx.bgp;
|
|
2085
|
+
if (!classification) {
|
|
2086
|
+
score += penalties.unknownClassification;
|
|
2087
|
+
reasons.push("ASN_CLASSIFICATION_UNKNOWN");
|
|
2088
|
+
return {
|
|
2089
|
+
score,
|
|
2090
|
+
reasons
|
|
2091
|
+
};
|
|
2092
|
+
}
|
|
2093
|
+
if (classification === "Content") {
|
|
2094
|
+
score += penalties.contentClassification;
|
|
2095
|
+
reasons.push("ASN_HOSTING_CLASSIFIED");
|
|
2096
|
+
}
|
|
2097
|
+
const hitsNum = parseInt(hits ?? "", 10);
|
|
2098
|
+
const isLowVisibility = Number.isFinite(hitsNum) && hitsNum >= 0 && hitsNum < penalties.lowVisibilityThreshold;
|
|
2099
|
+
if (isLowVisibility) {
|
|
2100
|
+
score += penalties.lowVisibilityPenalty;
|
|
2101
|
+
reasons.push("ASN_LOW_VISIBILITY");
|
|
2102
|
+
}
|
|
2103
|
+
if (classification === "Content" && isLowVisibility) {
|
|
2104
|
+
score += penalties.comboHostingLowVisibility;
|
|
2105
|
+
reasons.push("ASN_HOSTING_LOW_VISIBILITY_COMBO");
|
|
2106
|
+
}
|
|
2107
|
+
return {
|
|
2108
|
+
score,
|
|
2109
|
+
reasons
|
|
2110
|
+
};
|
|
2111
|
+
}
|
|
2112
|
+
};
|
|
2113
|
+
CheckerRegistry.register(new AsnClassificationChecker());
|
|
2114
|
+
|
|
2115
|
+
//#endregion
|
|
2116
|
+
//#region src/botDetector/checkers/torAnalysis.ts
|
|
2117
|
+
var TorAnalysisChecker = class {
|
|
2118
|
+
constructor() {
|
|
2119
|
+
this.name = "Tor Node Analysis";
|
|
2120
|
+
this.phase = "cheap";
|
|
2121
|
+
}
|
|
2122
|
+
isEnabled(config) {
|
|
2123
|
+
return config.checkers.enableTorAnalysis.enable;
|
|
2124
|
+
}
|
|
2125
|
+
parseFlags(flags) {
|
|
2126
|
+
if (!flags) return /* @__PURE__ */ new Set();
|
|
2127
|
+
return new Set(flags.split(",").map((f) => f.trim()));
|
|
2128
|
+
}
|
|
2129
|
+
canExitWebTraffic(summary) {
|
|
2130
|
+
if (!summary) return false;
|
|
2131
|
+
try {
|
|
2132
|
+
const policy = JSON.parse(summary);
|
|
2133
|
+
const coversWebPort = (entry) => {
|
|
2134
|
+
if (entry === "80" || entry === "443") return true;
|
|
2135
|
+
if (entry.includes("-")) {
|
|
2136
|
+
const [lo, hi] = entry.split("-").map(Number);
|
|
2137
|
+
return lo <= 80 && 80 <= hi || lo <= 443 && 443 <= hi;
|
|
2138
|
+
}
|
|
2139
|
+
return false;
|
|
2140
|
+
};
|
|
2141
|
+
if (policy.accept) return policy.accept.some(coversWebPort);
|
|
2142
|
+
if (policy.reject) return !policy.reject.some(coversWebPort);
|
|
2143
|
+
} catch {}
|
|
2144
|
+
return false;
|
|
2145
|
+
}
|
|
2146
|
+
run(ctx, config) {
|
|
2147
|
+
const checkConfig = config.checkers.enableTorAnalysis;
|
|
2148
|
+
const reasons = [];
|
|
2149
|
+
let score = 0;
|
|
2150
|
+
if (!checkConfig.enable) return {
|
|
2151
|
+
score,
|
|
2152
|
+
reasons
|
|
2153
|
+
};
|
|
2154
|
+
if (!ctx.tor || Object.keys(ctx.tor).length === 0) return {
|
|
2155
|
+
score,
|
|
2156
|
+
reasons
|
|
2157
|
+
};
|
|
2158
|
+
const { penalties } = checkConfig;
|
|
2159
|
+
const { running, exit_addresses, flags, recommended_version, version_status, exit_probability, exit_policy_summary, guard_probability } = ctx.tor;
|
|
2160
|
+
const flagSet = this.parseFlags(flags);
|
|
2161
|
+
if (running) {
|
|
2162
|
+
score += penalties.runningNode;
|
|
2163
|
+
reasons.push("TOR_ACTIVE_NODE");
|
|
2164
|
+
}
|
|
2165
|
+
if ((exit_addresses && exit_addresses.length > 0) ?? flagSet.has("Exit")) {
|
|
2166
|
+
score += penalties.exitNode + Math.ceil((exit_probability ?? 0) * 30);
|
|
2167
|
+
reasons.push("TOR_EXIT_NODE");
|
|
2168
|
+
if (this.canExitWebTraffic(exit_policy_summary)) {
|
|
2169
|
+
score += penalties.webExitCapable;
|
|
2170
|
+
reasons.push("TOR_WEB_EXIT_CAPABLE");
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
if (flagSet.has("BadExit")) {
|
|
2174
|
+
score += penalties.badExit;
|
|
2175
|
+
reasons.push("TOR_BAD_EXIT");
|
|
2176
|
+
}
|
|
2177
|
+
if (flagSet.has("Guard") || (guard_probability ?? 0) > 0) {
|
|
2178
|
+
score += penalties.guardNode;
|
|
2179
|
+
reasons.push("TOR_GUARD_NODE");
|
|
2180
|
+
}
|
|
2181
|
+
if (recommended_version === false || version_status === "obsolete") {
|
|
2182
|
+
score += penalties.obsoleteVersion;
|
|
2183
|
+
reasons.push("TOR_OBSOLETE_VERSION");
|
|
2184
|
+
}
|
|
2185
|
+
return {
|
|
2186
|
+
score,
|
|
2187
|
+
reasons
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
};
|
|
2191
|
+
CheckerRegistry.register(new TorAnalysisChecker());
|
|
2192
|
+
|
|
2193
|
+
//#endregion
|
|
2194
|
+
//#region src/botDetector/checkers/timezoneConsistency.ts
|
|
2195
|
+
var TimezoneConsistencyChecker = class {
|
|
2196
|
+
constructor() {
|
|
2197
|
+
this.name = "Timezone Consistency";
|
|
2198
|
+
this.phase = "cheap";
|
|
2199
|
+
}
|
|
2200
|
+
isEnabled(config) {
|
|
2201
|
+
return config.checkers.enableTimezoneConsistency.enable;
|
|
2202
|
+
}
|
|
2203
|
+
run(ctx, config) {
|
|
2204
|
+
const checkConfig = config.checkers.enableTimezoneConsistency;
|
|
2205
|
+
const reasons = [];
|
|
2206
|
+
let score = 0;
|
|
2207
|
+
if (!checkConfig.enable) return {
|
|
2208
|
+
score,
|
|
2209
|
+
reasons
|
|
2210
|
+
};
|
|
2211
|
+
const geoTimezone = ctx.geoData.timezone?.toLowerCase();
|
|
2212
|
+
if (!geoTimezone) return {
|
|
2213
|
+
score,
|
|
2214
|
+
reasons
|
|
2215
|
+
};
|
|
2216
|
+
const tzHeader = ctx.req.get("Sec-CH-UA-Timezone") ?? ctx.req.get("X-Timezone");
|
|
2217
|
+
if (tzHeader && tzHeader.toLowerCase() !== geoTimezone) {
|
|
2218
|
+
score += checkConfig.penalties;
|
|
2219
|
+
reasons.push("TZ_HEADER_GEO_MISMATCH");
|
|
2220
|
+
}
|
|
2221
|
+
return {
|
|
2222
|
+
score,
|
|
2223
|
+
reasons
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
};
|
|
2227
|
+
CheckerRegistry.register(new TimezoneConsistencyChecker());
|
|
2228
|
+
|
|
2229
|
+
//#endregion
|
|
2230
|
+
//#region src/botDetector/checkers/honeypot.ts
|
|
2231
|
+
var HoneypotChecker = class {
|
|
2232
|
+
constructor() {
|
|
2233
|
+
this.name = "Honeypot Path";
|
|
2234
|
+
this.phase = "cheap";
|
|
2235
|
+
}
|
|
2236
|
+
isEnabled(config) {
|
|
2237
|
+
return config.checkers.honeypot.enable;
|
|
2238
|
+
}
|
|
2239
|
+
run(ctx, config) {
|
|
2240
|
+
const checkConfig = config.checkers.honeypot;
|
|
2241
|
+
const reasons = [];
|
|
2242
|
+
if (!checkConfig.enable || checkConfig.paths.length === 0) return {
|
|
2243
|
+
score: 0,
|
|
2244
|
+
reasons
|
|
2245
|
+
};
|
|
2246
|
+
const requestPath = ctx.req.path.toLowerCase();
|
|
2247
|
+
if (checkConfig.paths.some((p) => requestPath === p.toLowerCase())) {
|
|
2248
|
+
reasons.push("HONEYPOT_PATH_HIT");
|
|
2249
|
+
reasons.push("BAD_BOT_DETECTED");
|
|
2250
|
+
}
|
|
2251
|
+
return {
|
|
2252
|
+
score: 0,
|
|
2253
|
+
reasons
|
|
2254
|
+
};
|
|
2255
|
+
}
|
|
2256
|
+
};
|
|
2257
|
+
CheckerRegistry.register(new HoneypotChecker());
|
|
2258
|
+
|
|
2259
|
+
//#endregion
|
|
2260
|
+
//#region src/botDetector/helpers/cache/sessionCache.ts
|
|
2261
|
+
const SESSION_TTL_SECONDS = 600;
|
|
2262
|
+
const PREFIX$3 = "session:";
|
|
2263
|
+
const sessionCache = {
|
|
2264
|
+
async get(sessionId) {
|
|
2265
|
+
const key = `${PREFIX$3}${sessionId}`;
|
|
2266
|
+
const storage = getStorage();
|
|
2267
|
+
const data = await storage.getItem(key);
|
|
2268
|
+
if (data) storage.setItem(key, data, { ttl: SESSION_TTL_SECONDS }).catch((err) => {
|
|
2269
|
+
consola.warn(`Failed to update session TTL for ${key}`, err);
|
|
2270
|
+
});
|
|
2271
|
+
return data;
|
|
2272
|
+
},
|
|
2273
|
+
async set(sessionId, entry) {
|
|
2274
|
+
const key = `${PREFIX$3}${sessionId}`;
|
|
2275
|
+
await getStorage().setItem(key, entry, { ttl: SESSION_TTL_SECONDS });
|
|
2276
|
+
},
|
|
2277
|
+
async delete(sessionId) {
|
|
2278
|
+
const key = `${PREFIX$3}${sessionId}`;
|
|
2279
|
+
await getStorage().removeItem(key);
|
|
2280
|
+
},
|
|
2281
|
+
async clear() {
|
|
2282
|
+
await getStorage().clear();
|
|
2283
|
+
}
|
|
2284
|
+
};
|
|
2285
|
+
|
|
2286
|
+
//#endregion
|
|
2287
|
+
//#region src/botDetector/checkers/sessionCoherence.ts
|
|
2288
|
+
var SessionCoherenceChecker = class {
|
|
2289
|
+
constructor() {
|
|
2290
|
+
this.name = "Session Coherence";
|
|
2291
|
+
this.phase = "heavy";
|
|
2292
|
+
}
|
|
2293
|
+
get logger() {
|
|
2294
|
+
this._logger ??= getLogger().child({
|
|
2295
|
+
service: "botDetector",
|
|
2296
|
+
branch: "checker",
|
|
2297
|
+
type: "SessionCoherenceChecker"
|
|
2298
|
+
});
|
|
2299
|
+
return this._logger;
|
|
2300
|
+
}
|
|
2301
|
+
isEnabled(config) {
|
|
2302
|
+
return config.checkers.enableSessionCoherence.enable;
|
|
2303
|
+
}
|
|
2304
|
+
async run(ctx, config) {
|
|
2305
|
+
const checkConfig = config.checkers.enableSessionCoherence;
|
|
2306
|
+
const reasons = [];
|
|
2307
|
+
let score = 0;
|
|
2308
|
+
if (!checkConfig.enable) return {
|
|
2309
|
+
score,
|
|
2310
|
+
reasons
|
|
2311
|
+
};
|
|
2312
|
+
if (!ctx.cookie) return {
|
|
2313
|
+
score,
|
|
2314
|
+
reasons
|
|
2315
|
+
};
|
|
2316
|
+
const currentPath = ctx.req.path;
|
|
2317
|
+
const refererHeader = ctx.req.get("Referer");
|
|
2318
|
+
const secFetchSite = ctx.req.get("Sec-Fetch-Site");
|
|
2319
|
+
const currentHostname = ctx.req.hostname;
|
|
2320
|
+
const cached = await sessionCache.get(ctx.cookie);
|
|
2321
|
+
if (secFetchSite === "same-origin" && !refererHeader || cached && !refererHeader) {
|
|
2322
|
+
score += checkConfig.penalties.missingReferer;
|
|
2323
|
+
reasons.push("SESSION_COHERENCE_MISSING_REFERER");
|
|
2324
|
+
} else if (refererHeader) try {
|
|
2325
|
+
const refererUrl = new URL(refererHeader);
|
|
2326
|
+
if (refererUrl.hostname !== currentHostname) {
|
|
2327
|
+
score += checkConfig.penalties.domainMismatch;
|
|
2328
|
+
reasons.push("SESSION_COHERENCE_DOMAIN_MISMATCH");
|
|
2329
|
+
} else if (cached && refererUrl.pathname !== cached.lastPath) {
|
|
2330
|
+
score += checkConfig.penalties.pathMismatch;
|
|
2331
|
+
reasons.push("SESSION_COHERENCE_PATH_MISMATCH");
|
|
2332
|
+
}
|
|
2333
|
+
} catch {
|
|
2334
|
+
score += checkConfig.penalties.missingReferer;
|
|
2335
|
+
reasons.push("SESSION_COHERENCE_INVALID_REFERER");
|
|
2336
|
+
}
|
|
2337
|
+
sessionCache.set(ctx.cookie, { lastPath: currentPath }).catch((err) => {
|
|
2338
|
+
this.logger.error({ err }, "Failed to save session in storage.");
|
|
2339
|
+
});
|
|
2340
|
+
return {
|
|
2341
|
+
score,
|
|
2342
|
+
reasons
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
};
|
|
2346
|
+
CheckerRegistry.register(new SessionCoherenceChecker());
|
|
2347
|
+
|
|
2348
|
+
//#endregion
|
|
2349
|
+
//#region src/botDetector/helpers/cache/timingCache.ts
|
|
2350
|
+
const TIMING_TTL_SECONDS = 900;
|
|
2351
|
+
const PREFIX$2 = "timing:";
|
|
2352
|
+
const timingCache = {
|
|
2353
|
+
async get(visitorId) {
|
|
2354
|
+
const key = `${PREFIX$2}${visitorId}`;
|
|
2355
|
+
return await getStorage().getItem(key);
|
|
2356
|
+
},
|
|
2357
|
+
async set(visitorId, entry) {
|
|
2358
|
+
const key = `${PREFIX$2}${visitorId}`;
|
|
2359
|
+
await getStorage().setItem(key, entry, { ttl: TIMING_TTL_SECONDS });
|
|
2360
|
+
},
|
|
2361
|
+
async delete(visitorId) {
|
|
2362
|
+
const key = `${PREFIX$2}${visitorId}`;
|
|
2363
|
+
await getStorage().removeItem(key);
|
|
2364
|
+
},
|
|
2365
|
+
async clear() {
|
|
2366
|
+
await getStorage().clear();
|
|
2367
|
+
}
|
|
2368
|
+
};
|
|
2369
|
+
|
|
2370
|
+
//#endregion
|
|
2371
|
+
//#region src/botDetector/checkers/velocityFingerprint.ts
|
|
2372
|
+
const MAX_SAMPLES = 10;
|
|
2373
|
+
const MIN_SAMPLES_TO_EVALUATE = 5;
|
|
2374
|
+
var VelocityFingerprintChecker = class {
|
|
2375
|
+
constructor() {
|
|
2376
|
+
this.name = "Velocity Fingerprinting";
|
|
2377
|
+
this.phase = "heavy";
|
|
2378
|
+
}
|
|
2379
|
+
get logger() {
|
|
2380
|
+
this._logger ??= getLogger().child({
|
|
2381
|
+
service: "botDetector",
|
|
2382
|
+
branch: "checker",
|
|
2383
|
+
type: "VelocityFingerprintChecker"
|
|
2384
|
+
});
|
|
2385
|
+
return this._logger;
|
|
2386
|
+
}
|
|
2387
|
+
isEnabled(config) {
|
|
2388
|
+
return config.checkers.enableVelocityFingerprint.enable;
|
|
2389
|
+
}
|
|
2390
|
+
async run(ctx, config) {
|
|
2391
|
+
const checkConfig = config.checkers.enableVelocityFingerprint;
|
|
2392
|
+
const reasons = [];
|
|
2393
|
+
let score = 0;
|
|
2394
|
+
if (!checkConfig.enable || !ctx.cookie) return {
|
|
2395
|
+
score,
|
|
2396
|
+
reasons
|
|
2397
|
+
};
|
|
2398
|
+
const now = Date.now();
|
|
2399
|
+
const timestamps = [...await timingCache.get(ctx.cookie) ?? [], now].slice(-MAX_SAMPLES);
|
|
2400
|
+
timingCache.set(ctx.cookie, timestamps).catch((err) => {
|
|
2401
|
+
this.logger.error({ err }, "Failed to save timingCache in storage");
|
|
2402
|
+
});
|
|
2403
|
+
if (timestamps.length < MIN_SAMPLES_TO_EVALUATE) return {
|
|
2404
|
+
score,
|
|
2405
|
+
reasons
|
|
2406
|
+
};
|
|
2407
|
+
const intervals = [];
|
|
2408
|
+
for (let i = 1; i < timestamps.length; i++) intervals.push(timestamps[i] - timestamps[i - 1]);
|
|
2409
|
+
const mean = intervals.reduce((a, b) => a + b, 0) / intervals.length;
|
|
2410
|
+
if (mean === 0) return {
|
|
2411
|
+
score,
|
|
2412
|
+
reasons
|
|
2413
|
+
};
|
|
2414
|
+
const variance = intervals.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / intervals.length;
|
|
2415
|
+
if (Math.sqrt(variance) / mean < checkConfig.cvThreshold) {
|
|
2416
|
+
score += checkConfig.penalties;
|
|
2417
|
+
reasons.push("TIMING_TOO_REGULAR");
|
|
2418
|
+
}
|
|
2419
|
+
return {
|
|
2420
|
+
score,
|
|
2421
|
+
reasons
|
|
2422
|
+
};
|
|
2423
|
+
}
|
|
2424
|
+
};
|
|
2425
|
+
CheckerRegistry.register(new VelocityFingerprintChecker());
|
|
2426
|
+
|
|
2427
|
+
//#endregion
|
|
2428
|
+
//#region src/botDetector/checkers/knownBadIps.ts
|
|
2429
|
+
var KnownBadIps = class {
|
|
2430
|
+
constructor() {
|
|
2431
|
+
this.name = "KnownBadIps";
|
|
2432
|
+
this.phase = "cheap";
|
|
2433
|
+
}
|
|
2434
|
+
isEnabled(config) {
|
|
2435
|
+
return config.checkers.enableKnownBadIpsCheck.enable;
|
|
2436
|
+
}
|
|
2437
|
+
run(ctx, config) {
|
|
2438
|
+
const { enableKnownBadIpsCheck } = config.checkers;
|
|
2439
|
+
const reasons = [];
|
|
2440
|
+
let score = 0;
|
|
2441
|
+
if (!enableKnownBadIpsCheck.enable) return {
|
|
2442
|
+
score,
|
|
2443
|
+
reasons
|
|
2444
|
+
};
|
|
2445
|
+
const ds = getDataSources();
|
|
2446
|
+
const ip = ctx.ipAddress;
|
|
2447
|
+
if (ds.bannedDataBase(ip)) {
|
|
2448
|
+
reasons.push("PREVIOUSLY_BANNED_IP", "BAD_BOT_DETECTED");
|
|
2449
|
+
return {
|
|
2450
|
+
score,
|
|
2451
|
+
reasons
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
const highRisk = ds.highRiskDataBase(ip);
|
|
2455
|
+
if (highRisk) {
|
|
2456
|
+
const ratio = Math.min(highRisk.score / config.banScore, 1);
|
|
2457
|
+
score += Math.round(enableKnownBadIpsCheck.highRiskPenalty * ratio);
|
|
2458
|
+
reasons.push("PREVIOUSLY_HIGH_RISK_IP");
|
|
2459
|
+
}
|
|
2460
|
+
return {
|
|
2461
|
+
score,
|
|
2462
|
+
reasons
|
|
2463
|
+
};
|
|
2464
|
+
}
|
|
2465
|
+
};
|
|
2466
|
+
CheckerRegistry.register(new KnownBadIps());
|
|
2467
|
+
|
|
2468
|
+
//#endregion
|
|
2469
|
+
//#region src/botDetector/helpers/exceptions.ts
|
|
2470
|
+
var BadBotDetected = class extends Error {
|
|
2471
|
+
constructor(message = "Bad bot detected") {
|
|
2472
|
+
super(message);
|
|
2473
|
+
this.name = "BadBotDetected";
|
|
2474
|
+
}
|
|
2475
|
+
};
|
|
2476
|
+
var GoodBotDetected = class extends Error {
|
|
2477
|
+
constructor(message = "Good bot detected") {
|
|
2478
|
+
super(message);
|
|
2479
|
+
this.name = "GoodBotDetected";
|
|
2480
|
+
}
|
|
2481
|
+
};
|
|
2482
|
+
|
|
2483
|
+
//#endregion
|
|
2484
|
+
//#region src/botDetector/penalties/banIP.ts
|
|
2485
|
+
const UFW = "/usr/sbin/ufw";
|
|
2486
|
+
function banIp(ip, info) {
|
|
2487
|
+
const log = getLogger().child({
|
|
2488
|
+
service: "BOT DETECTOR",
|
|
2489
|
+
branch: `banIp`,
|
|
2490
|
+
ipAddress: ip,
|
|
2491
|
+
details: info
|
|
2492
|
+
});
|
|
2493
|
+
const { punishmentType } = getConfiguration();
|
|
2494
|
+
if (!punishmentType.enableFireWallBan) {
|
|
2495
|
+
log.info(`Firewall punishment ban is disabled, skipping`);
|
|
2496
|
+
return;
|
|
2497
|
+
}
|
|
2498
|
+
log.info("about to ban an IP");
|
|
2499
|
+
return new Promise((resolve, reject) => {
|
|
2500
|
+
const child = spawn("sudo", [
|
|
2501
|
+
"-n",
|
|
2502
|
+
UFW,
|
|
2503
|
+
"insert",
|
|
2504
|
+
"1",
|
|
2505
|
+
"deny",
|
|
2506
|
+
"from",
|
|
2507
|
+
ip
|
|
2508
|
+
], {
|
|
2509
|
+
stdio: [
|
|
2510
|
+
"ignore",
|
|
2511
|
+
"ignore",
|
|
2512
|
+
"pipe"
|
|
2513
|
+
],
|
|
2514
|
+
detached: true
|
|
2515
|
+
});
|
|
2516
|
+
child.unref();
|
|
2517
|
+
let stderr = "";
|
|
2518
|
+
const timer = setTimeout(() => {
|
|
2519
|
+
child.kill("SIGKILL");
|
|
2520
|
+
log.warn(`ufw hang timeout on ${ip}`);
|
|
2521
|
+
reject(/* @__PURE__ */ new Error(`ufw hang timeout on ${ip}`));
|
|
2522
|
+
}, 5e3);
|
|
2523
|
+
child.stderr.on("data", (d) => stderr += String(d));
|
|
2524
|
+
child.on("error", (err) => {
|
|
2525
|
+
clearTimeout(timer);
|
|
2526
|
+
log.fatal({ err }, `- CRITICAL - UFW spawn failed`);
|
|
2527
|
+
reject(err);
|
|
2528
|
+
});
|
|
2529
|
+
child.on("close", (code) => {
|
|
2530
|
+
clearTimeout(timer);
|
|
2531
|
+
if (code !== 0) {
|
|
2532
|
+
log.fatal({ code }, `- CRITICAL - UFW ban failed`);
|
|
2533
|
+
reject(/* @__PURE__ */ new Error(`ufw exited ${String(code)}`));
|
|
2534
|
+
return;
|
|
2535
|
+
}
|
|
2536
|
+
log.info(`Banning Detected Bot/Malicious User. IP ${ip} banned (score ${String(info.score)})\nReasons:\n- ${info.reasons.join("\n- ")}`);
|
|
2537
|
+
resolve();
|
|
2538
|
+
});
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
//#endregion
|
|
2543
|
+
//#region src/botDetector/helpers/processChecks.ts
|
|
2544
|
+
async function processChecks(checkers, ctx, config, botScore, reasons, phaseLabel = "phase") {
|
|
2545
|
+
const { banScore } = getConfiguration();
|
|
2546
|
+
const log = getLogger().child({
|
|
2547
|
+
service: `BOT DETECTOR`,
|
|
2548
|
+
branch: "checks"
|
|
2549
|
+
});
|
|
2550
|
+
const reqId = Date.now();
|
|
2551
|
+
const phaseStart = performance.now();
|
|
2552
|
+
log.info({
|
|
2553
|
+
phase: phaseLabel,
|
|
2554
|
+
reqId,
|
|
2555
|
+
event: "start"
|
|
2556
|
+
});
|
|
2557
|
+
const banLimit = banScore;
|
|
2558
|
+
for (const checker of checkers) {
|
|
2559
|
+
const label = checker.name;
|
|
2560
|
+
const checkStart = performance.now();
|
|
2561
|
+
log.info({
|
|
2562
|
+
reqId,
|
|
2563
|
+
check: label,
|
|
2564
|
+
event: "start"
|
|
2565
|
+
});
|
|
2566
|
+
const { score, reasons: rs } = await checker.run(ctx, config);
|
|
2567
|
+
const checkEnd = performance.now();
|
|
2568
|
+
log.info({
|
|
2569
|
+
reqId,
|
|
2570
|
+
check: label,
|
|
2571
|
+
event: "end",
|
|
2572
|
+
durationMs: +(checkEnd - checkStart).toFixed(3),
|
|
2573
|
+
score,
|
|
2574
|
+
reasons: rs
|
|
2575
|
+
});
|
|
2576
|
+
botScore += score;
|
|
2577
|
+
rs.forEach((r) => reasons.push(r));
|
|
2578
|
+
if (rs.includes("GOOD_BOT_IDENTIFIED")) throw new GoodBotDetected();
|
|
2579
|
+
if (rs.includes("BAD_BOT_DETECTED")) throw new BadBotDetected();
|
|
2580
|
+
if (botScore >= banLimit) {
|
|
2581
|
+
log.warn({
|
|
2582
|
+
reqId,
|
|
2583
|
+
botScore
|
|
2584
|
+
}, "Bot detected — aborting checks");
|
|
2585
|
+
break;
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
const phaseEnd = performance.now();
|
|
2589
|
+
log.info({
|
|
2590
|
+
reqId,
|
|
2591
|
+
phase: phaseLabel,
|
|
2592
|
+
event: "end",
|
|
2593
|
+
durationMs: +(phaseEnd - phaseStart).toFixed(3),
|
|
2594
|
+
Score: botScore
|
|
2595
|
+
});
|
|
2596
|
+
return botScore;
|
|
2597
|
+
}
|
|
2598
|
+
|
|
2599
|
+
//#endregion
|
|
2600
|
+
//#region src/botDetector/helpers/cache/reputationCache.ts
|
|
2601
|
+
const PREFIX$1 = "reputation:";
|
|
2602
|
+
const reputationCache = {
|
|
2603
|
+
async get(cookie) {
|
|
2604
|
+
return getStorage().getItem(`${PREFIX$1}${cookie}`);
|
|
2605
|
+
},
|
|
2606
|
+
async set(cookie, entry) {
|
|
2607
|
+
const { checksTimeRateControl } = getConfiguration();
|
|
2608
|
+
await getStorage().setItem(`${PREFIX$1}${cookie}`, entry, { ttl: Math.floor(checksTimeRateControl.checkEvery / 1e3) });
|
|
2609
|
+
},
|
|
2610
|
+
async delete(cookie) {
|
|
2611
|
+
await getStorage().removeItem(`${PREFIX$1}${cookie}`);
|
|
2612
|
+
},
|
|
2613
|
+
async clear() {
|
|
2614
|
+
await getStorage().clear();
|
|
2615
|
+
}
|
|
2616
|
+
};
|
|
2617
|
+
|
|
2618
|
+
//#endregion
|
|
2619
|
+
//#region src/botDetector.ts
|
|
2620
|
+
async function uaAndGeoBotDetector(req, ipAddress, userAgent, geo, parsedUA, buildCustomContext) {
|
|
2621
|
+
const log = getLogger().child({
|
|
2622
|
+
service: "BOT DETECTOR",
|
|
2623
|
+
branch: "main"
|
|
2624
|
+
});
|
|
2625
|
+
const { banScore, maxScore, setNewComputedScore } = getConfiguration();
|
|
2626
|
+
const BAN_THRESHOLD = banScore;
|
|
2627
|
+
const MAX_SCORE = maxScore;
|
|
2628
|
+
const reasons = [];
|
|
2629
|
+
let botScore = 0;
|
|
2630
|
+
const cookie = req.cookies.canary_id;
|
|
2631
|
+
const uaString = req.get("User-Agent") ?? "";
|
|
2632
|
+
log.info(`BotDetection called for ${req.method} ${req.get("X-Forwarded-Host") ?? ""}`);
|
|
2633
|
+
const threatsLevels = getDataSources().fireholLvl1DataBase(ipAddress) ? 1 : getDataSources().fireholLvl2DataBase(ipAddress) ? 2 : getDataSources().fireholLvl3DataBase(ipAddress) ? 3 : getDataSources().fireholLvl4DataBase(ipAddress) ? 4 : null;
|
|
2634
|
+
const proxy = () => {
|
|
2635
|
+
const isProxy = getDataSources().proxyDataBase(ipAddress);
|
|
2636
|
+
if (isProxy) return {
|
|
2637
|
+
proxy: true,
|
|
2638
|
+
proxyType: isProxy.comment
|
|
2639
|
+
};
|
|
2640
|
+
return { proxy: false };
|
|
2641
|
+
};
|
|
2642
|
+
const { tor, asn, threatLevel, anon } = {
|
|
2643
|
+
tor: getDataSources().torDataBase(ipAddress),
|
|
2644
|
+
asn: getDataSources().asnDataBase(ipAddress),
|
|
2645
|
+
threatLevel: threatsLevels,
|
|
2646
|
+
anon: getDataSources().fireholAnonDataBase(ipAddress) ? true : false
|
|
2647
|
+
};
|
|
2648
|
+
const proxyResult = proxy();
|
|
2649
|
+
const ctx = {
|
|
2650
|
+
req,
|
|
2651
|
+
ipAddress,
|
|
2652
|
+
parsedUA,
|
|
2653
|
+
geoData: geo,
|
|
2654
|
+
cookie,
|
|
2655
|
+
proxy: {
|
|
2656
|
+
isProxy: proxyResult.proxy,
|
|
2657
|
+
proxyType: proxyResult.proxyType
|
|
2658
|
+
},
|
|
2659
|
+
anon,
|
|
2660
|
+
bgp: asn ?? {},
|
|
2661
|
+
tor: tor ?? {},
|
|
2662
|
+
threatLevel,
|
|
2663
|
+
custom: buildCustomContext ? buildCustomContext(req) : {}
|
|
2664
|
+
};
|
|
2665
|
+
const cheapChecks = CheckerRegistry.getEnabled("cheap", getConfiguration());
|
|
2666
|
+
const checks = CheckerRegistry.getEnabled("heavy", getConfiguration());
|
|
2667
|
+
try {
|
|
2668
|
+
botScore = await processChecks(cheapChecks, ctx, getConfiguration(), botScore, reasons, "cheapPhase");
|
|
2669
|
+
if (botScore < BAN_THRESHOLD) botScore = await processChecks(checks, ctx, getConfiguration(), botScore, reasons, "heavyPhase");
|
|
2670
|
+
} catch (error) {
|
|
2671
|
+
if (error instanceof BadBotDetected) {
|
|
2672
|
+
const bannedInfo = {
|
|
2673
|
+
score: BAN_THRESHOLD,
|
|
2674
|
+
reasons: Array.from(reasons)
|
|
2675
|
+
};
|
|
2676
|
+
banIp(ipAddress, bannedInfo);
|
|
2677
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "update_banned_ip", {
|
|
2678
|
+
cookie: cookie ?? "",
|
|
2679
|
+
ipAddress,
|
|
2680
|
+
country: ctx.geoData.country ?? "",
|
|
2681
|
+
user_agent: uaString,
|
|
2682
|
+
info: bannedInfo
|
|
2683
|
+
}, "deferred");
|
|
2684
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "is_bot_update", {
|
|
2685
|
+
isBot: true,
|
|
2686
|
+
cookie: cookie ?? ""
|
|
2687
|
+
}, "deferred");
|
|
2688
|
+
return true;
|
|
2689
|
+
}
|
|
2690
|
+
if (error instanceof GoodBotDetected) return false;
|
|
2691
|
+
log.error({ error }, "unexpected error");
|
|
2692
|
+
}
|
|
2693
|
+
botScore = Math.min(botScore, MAX_SCORE);
|
|
2694
|
+
if (botScore >= BAN_THRESHOLD) {
|
|
2695
|
+
log.info(`Starting Ban for ${ipAddress} ${userAgent}`);
|
|
2696
|
+
const bannedInfo = {
|
|
2697
|
+
score: botScore,
|
|
2698
|
+
reasons: Array.from(reasons)
|
|
2699
|
+
};
|
|
2700
|
+
banIp(ipAddress, bannedInfo);
|
|
2701
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "update_banned_ip", {
|
|
2702
|
+
cookie: cookie ?? "",
|
|
2703
|
+
ipAddress,
|
|
2704
|
+
country: ctx.geoData.country ?? "",
|
|
2705
|
+
user_agent: uaString,
|
|
2706
|
+
info: bannedInfo
|
|
2707
|
+
}, "deferred");
|
|
2708
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "is_bot_update", {
|
|
2709
|
+
isBot: true,
|
|
2710
|
+
cookie: cookie ?? ""
|
|
2711
|
+
}, "deferred");
|
|
2712
|
+
return true;
|
|
2713
|
+
}
|
|
2714
|
+
if (setNewComputedScore) {
|
|
2715
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "score_update", {
|
|
2716
|
+
score: botScore,
|
|
2717
|
+
cookie: cookie ?? ""
|
|
2718
|
+
}, "deferred");
|
|
2719
|
+
reputationCache.set(cookie ?? "", {
|
|
2720
|
+
isBot: false,
|
|
2721
|
+
score: botScore
|
|
2722
|
+
}).catch((err) => {
|
|
2723
|
+
log.error({ err }, "Failed to set reputationCache in storage");
|
|
2724
|
+
});
|
|
2725
|
+
} else {
|
|
2726
|
+
const cached = await reputationCache.get(cookie ?? "");
|
|
2727
|
+
if (!cached || cached.score === 0) {
|
|
2728
|
+
getBatchQueue().addQueue(cookie ?? "", ipAddress, "score_update", {
|
|
2729
|
+
score: botScore,
|
|
2730
|
+
cookie: cookie ?? ""
|
|
2731
|
+
}, "deferred");
|
|
2732
|
+
reputationCache.set(cookie ?? "", {
|
|
2733
|
+
isBot: false,
|
|
2734
|
+
score: botScore
|
|
2735
|
+
}).catch((err) => {
|
|
2736
|
+
log.error({ err }, "Failed to set reputationCache in storage");
|
|
2737
|
+
});
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
return false;
|
|
2741
|
+
}
|
|
2742
|
+
|
|
2743
|
+
//#endregion
|
|
2744
|
+
//#region src/botDetector/helpers/cache/cannaryCache.ts
|
|
2745
|
+
const PREFIX = "visitor:";
|
|
2746
|
+
const visitorCache = {
|
|
2747
|
+
async get(cookie) {
|
|
2748
|
+
return getStorage().getItem(`${PREFIX}${cookie}`);
|
|
2749
|
+
},
|
|
2750
|
+
async set(cookie, entry) {
|
|
2751
|
+
const { checksTimeRateControl } = getConfiguration();
|
|
2752
|
+
await getStorage().setItem(`${PREFIX}${cookie}`, entry, { ttl: Math.floor(checksTimeRateControl.checkEvery / 1e3) });
|
|
2753
|
+
},
|
|
2754
|
+
async delete(cookie) {
|
|
2755
|
+
await getStorage().removeItem(`${PREFIX}${cookie}`);
|
|
2756
|
+
},
|
|
2757
|
+
async clear() {
|
|
2758
|
+
await getStorage().clear();
|
|
2759
|
+
}
|
|
2760
|
+
};
|
|
2761
|
+
|
|
2762
|
+
//#endregion
|
|
2763
|
+
//#region src/botDetector/helpers/reputation.ts
|
|
2764
|
+
async function userReputation(cookie) {
|
|
2765
|
+
const log = getLogger().child({
|
|
2766
|
+
service: `BOT DETECTOR`,
|
|
2767
|
+
branch: `reputation`
|
|
2768
|
+
});
|
|
2769
|
+
const db = getDb();
|
|
2770
|
+
const { banScore, restoredReputationPoints, setNewComputedScore } = getConfiguration();
|
|
2771
|
+
const botScore = banScore;
|
|
2772
|
+
const cached = await reputationCache.get(cookie);
|
|
2773
|
+
if (cached) {
|
|
2774
|
+
log.info(`CACHE HIT cookie=${cookie} score=${String(cached.score)} → (botScore=${String(botScore)})`);
|
|
2775
|
+
if (cached.isBot) return;
|
|
2776
|
+
if (!cached.isBot && cached.score > 0 && cached.score < botScore) {
|
|
2777
|
+
log.info(`updating cache score cookie=${cookie} score=${String(cached.score)} → (botScore=${String(botScore)})`);
|
|
2778
|
+
const newReputation = Math.max(0, cached.score - restoredReputationPoints);
|
|
2779
|
+
if (newReputation !== cached.score) {
|
|
2780
|
+
getBatchQueue().addQueue(cookie, "", "score_update", {
|
|
2781
|
+
score: newReputation,
|
|
2782
|
+
cookie
|
|
2783
|
+
}, "deferred");
|
|
2784
|
+
log.info(`updating cache score to DB cookie=${cookie} score=${String(cached.score)} → (botScore=${String(botScore)})`);
|
|
2785
|
+
reputationCache.set(cookie, {
|
|
2786
|
+
isBot: cached.isBot,
|
|
2787
|
+
score: newReputation
|
|
2788
|
+
}).catch((err) => {
|
|
2789
|
+
log.error({ err }, "Failed to save reputationCache in storage");
|
|
2790
|
+
});
|
|
2791
|
+
}
|
|
2792
|
+
log.info(`finished Updating Score from cache to DB cookie: ${cookie} NEW SCORE: ${String(newReputation)}`);
|
|
2793
|
+
}
|
|
2794
|
+
return;
|
|
2795
|
+
}
|
|
2796
|
+
try {
|
|
2797
|
+
const visitor = await prep(db, `
|
|
2798
|
+
SELECT is_bot,
|
|
2799
|
+
suspicious_activity_score
|
|
2800
|
+
FROM visitors
|
|
2801
|
+
WHERE canary_id = ?
|
|
2802
|
+
LIMIT 1`).get(cookie);
|
|
2803
|
+
if (!visitor) {
|
|
2804
|
+
log.warn(`no visitor record for canary_id=${cookie}`);
|
|
2805
|
+
return;
|
|
2806
|
+
}
|
|
2807
|
+
const isBot = visitor.is_bot === 0 ? false : true;
|
|
2808
|
+
const reputation = visitor.suspicious_activity_score;
|
|
2809
|
+
if (isBot) return;
|
|
2810
|
+
if (!setNewComputedScore) reputationCache.set(cookie, {
|
|
2811
|
+
isBot,
|
|
2812
|
+
score: reputation
|
|
2813
|
+
}).catch((err) => {
|
|
2814
|
+
log.error({ err }, "Failed to save reputationCache in storage");
|
|
2815
|
+
});
|
|
2816
|
+
log.info({
|
|
2817
|
+
label: "[REP-GATE]",
|
|
2818
|
+
isBot,
|
|
2819
|
+
score: reputation,
|
|
2820
|
+
"score>0": reputation > 0,
|
|
2821
|
+
"score<ban": reputation < botScore,
|
|
2822
|
+
healPts: restoredReputationPoints
|
|
2823
|
+
});
|
|
2824
|
+
if (!isBot && reputation > 0 && reputation < botScore) {
|
|
2825
|
+
log.info(`calculating new score cookie=${cookie} score=${String(reputation)} → (botScore=${String(botScore)})`);
|
|
2826
|
+
const newReputation = Math.max(0, reputation - restoredReputationPoints);
|
|
2827
|
+
if (newReputation !== reputation) {
|
|
2828
|
+
getBatchQueue().addQueue(cookie, "", "score_update", {
|
|
2829
|
+
score: newReputation,
|
|
2830
|
+
cookie
|
|
2831
|
+
}, "deferred");
|
|
2832
|
+
log.info(`Update Score for cookie', ${cookie}, 'New Score:', ${String(newReputation)}`);
|
|
2833
|
+
reputationCache.set(cookie, {
|
|
2834
|
+
isBot,
|
|
2835
|
+
score: newReputation
|
|
2836
|
+
}).catch((err) => {
|
|
2837
|
+
log.error({ err }, "Failed to save reputationCache in storage");
|
|
2838
|
+
});
|
|
2839
|
+
}
|
|
2840
|
+
}
|
|
2841
|
+
} catch (err) {
|
|
2842
|
+
log.error({ err }, `An error occurred updating visitor reputation`);
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2846
|
+
//#endregion
|
|
2847
|
+
//#region src/botDetector/utils/whitelist.ts
|
|
2848
|
+
function isInWhiteList(ipAddress) {
|
|
2849
|
+
const { whiteList } = getConfiguration();
|
|
2850
|
+
if (whiteList) return whiteList.includes(ipAddress);
|
|
2851
|
+
return false;
|
|
2852
|
+
}
|
|
2853
|
+
|
|
2854
|
+
//#endregion
|
|
2855
|
+
//#region src/botDetector/utils/nowMysql.ts
|
|
2856
|
+
const nowMysql = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 19).replace("T", " ");
|
|
2857
|
+
|
|
2858
|
+
//#endregion
|
|
2859
|
+
//#region src/botDetector/middlewares/canaryCookieChecker.ts
|
|
2860
|
+
function validator(buildCustomContext) {
|
|
2861
|
+
return async (req, res, next) => {
|
|
2862
|
+
const { checksTimeRateControl } = getConfiguration();
|
|
2863
|
+
let canary = req.cookies.canary_id;
|
|
2864
|
+
const ua = req.get("User-Agent") ?? "";
|
|
2865
|
+
const ip = req.ip ?? "";
|
|
2866
|
+
const log = getLogger().child({
|
|
2867
|
+
service: "BOT DETECTOR",
|
|
2868
|
+
branch: `main`,
|
|
2869
|
+
canary
|
|
2870
|
+
});
|
|
2871
|
+
log.info(`Validator entered for ${req.method} ${req.get("X-Forwarded-Host") ?? ""}`);
|
|
2872
|
+
log.info({ cookies: req.cookies }, `Incoming cookies`);
|
|
2873
|
+
const whiteList = isInWhiteList(ip);
|
|
2874
|
+
if (canary) {
|
|
2875
|
+
const cached = await visitorCache.get(canary);
|
|
2876
|
+
if (cached) {
|
|
2877
|
+
if (cached.banned && !whiteList) {
|
|
2878
|
+
res.sendStatus(403);
|
|
2879
|
+
return;
|
|
2880
|
+
}
|
|
2881
|
+
req.newVisitorId = cached.visitor_id;
|
|
2882
|
+
if (!checksTimeRateControl.checkEveryRequest) {
|
|
2883
|
+
next();
|
|
2884
|
+
return;
|
|
2885
|
+
}
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2888
|
+
if (!canary) {
|
|
2889
|
+
log.info(`No canary_id cookie found. Generating a new one.`);
|
|
2890
|
+
const cookieValue = randomBytes(32).toString("hex");
|
|
2891
|
+
canary = cookieValue;
|
|
2892
|
+
makeCookie(res, "canary_id", cookieValue, {
|
|
2893
|
+
httpOnly: true,
|
|
2894
|
+
sameSite: "lax",
|
|
2895
|
+
maxAge: 1e3 * 60 * 60 * 24 * 90,
|
|
2896
|
+
secure: true,
|
|
2897
|
+
path: "/"
|
|
2898
|
+
});
|
|
2899
|
+
req.cookies.canary_id = canary;
|
|
2900
|
+
log.info(`New canary_id cookie set:, ${cookieValue}`);
|
|
2901
|
+
}
|
|
2902
|
+
const geo = getData(ip);
|
|
2903
|
+
const parsedUA = parseUA(ua);
|
|
2904
|
+
const visitorId = randomUUID();
|
|
2905
|
+
const userValidation = {
|
|
2906
|
+
visitorId,
|
|
2907
|
+
cookie: canary,
|
|
2908
|
+
userAgent: ua,
|
|
2909
|
+
ipAddress: ip,
|
|
2910
|
+
device_type: parsedUA.device,
|
|
2911
|
+
browser: parsedUA.browser,
|
|
2912
|
+
is_bot: false,
|
|
2913
|
+
first_seen: nowMysql(),
|
|
2914
|
+
last_seen: nowMysql(),
|
|
2915
|
+
request_count: 1,
|
|
2916
|
+
deviceVendor: parsedUA.deviceVendor,
|
|
2917
|
+
deviceModel: parsedUA.deviceModel,
|
|
2918
|
+
browserType: parsedUA.browserType,
|
|
2919
|
+
browserVersion: parsedUA.browserVersion,
|
|
2920
|
+
os: parsedUA.os,
|
|
2921
|
+
activity_score: "0",
|
|
2922
|
+
...geo
|
|
2923
|
+
};
|
|
2924
|
+
getBatchQueue().addQueue(canary, ip, "visitor_upsert", { insert: userValidation }, "deferred");
|
|
2925
|
+
req.newVisitorId = visitorId;
|
|
2926
|
+
if (whiteList) {
|
|
2927
|
+
log.info(`${ip} is in white list skipping botDetection checks.`);
|
|
2928
|
+
visitorCache.set(canary, {
|
|
2929
|
+
banned: false,
|
|
2930
|
+
visitor_id: visitorId
|
|
2931
|
+
}).catch((err) => {
|
|
2932
|
+
log.error({ err }, "Failed to save visitorCache in storage");
|
|
2933
|
+
});
|
|
2934
|
+
req.botDetection = {
|
|
2935
|
+
success: true,
|
|
2936
|
+
banned: false,
|
|
2937
|
+
time: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2938
|
+
ipAddress: ip
|
|
2939
|
+
};
|
|
2940
|
+
next();
|
|
2941
|
+
return;
|
|
2942
|
+
}
|
|
2943
|
+
const brvConfig = getConfiguration().checkers.enableBehaviorRateCheck;
|
|
2944
|
+
if (brvConfig.enable) {
|
|
2945
|
+
if (!await rateCache.get(canary)) {
|
|
2946
|
+
const ttlSeconds = Math.ceil(brvConfig.behavioral_window / 1e3);
|
|
2947
|
+
rateCache.set(canary, {
|
|
2948
|
+
score: 0,
|
|
2949
|
+
timestamp: Date.now(),
|
|
2950
|
+
request_count: 1
|
|
2951
|
+
}, ttlSeconds).catch((err) => {
|
|
2952
|
+
log.error({ err }, "Failed to pre seed rateCache");
|
|
2953
|
+
});
|
|
2954
|
+
}
|
|
2955
|
+
}
|
|
2956
|
+
const isBot = await uaAndGeoBotDetector(req, ip, ua, geo, parsedUA, buildCustomContext);
|
|
2957
|
+
visitorCache.set(canary, {
|
|
2958
|
+
banned: isBot,
|
|
2959
|
+
visitor_id: visitorId
|
|
2960
|
+
}).catch((err) => {
|
|
2961
|
+
log.error({ err }, "Failed to save visitorCache in storage");
|
|
2962
|
+
});
|
|
2963
|
+
if (isBot) {
|
|
2964
|
+
res.sendStatus(403);
|
|
2965
|
+
return;
|
|
2966
|
+
}
|
|
2967
|
+
req.botDetection = {
|
|
2968
|
+
success: true,
|
|
2969
|
+
banned: isBot,
|
|
2970
|
+
time: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2971
|
+
ipAddress: ip
|
|
2972
|
+
};
|
|
2973
|
+
userReputation(canary).catch((err) => {
|
|
2974
|
+
consola.error("[BOT DETECTION - MIDDLEWARE] userReputation failed:", err);
|
|
2975
|
+
});
|
|
2976
|
+
next();
|
|
2977
|
+
};
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
//#endregion
|
|
2981
|
+
//#region src/botDetector/routes/visitorLog.ts
|
|
2982
|
+
const router = Router();
|
|
2983
|
+
router.use("/check", validator(), (req, res) => {
|
|
2984
|
+
res.json({
|
|
2985
|
+
results: req.botDetection,
|
|
2986
|
+
message: "Fingerprint logged successfully"
|
|
2987
|
+
});
|
|
2988
|
+
});
|
|
2989
|
+
|
|
2990
|
+
//#endregion
|
|
2991
|
+
//#region src/botDetector/db/generator.ts
|
|
2992
|
+
const MMDB_DIR = path.resolve(getLibraryRoot(), "_data-sources");
|
|
2993
|
+
async function buildBannedMmdb(generateTypes, mmdbctlPath) {
|
|
2994
|
+
const log = getLogger().child({
|
|
2995
|
+
service: "BOT DETECTOR",
|
|
2996
|
+
branch: "generator",
|
|
2997
|
+
db: "banned"
|
|
2998
|
+
});
|
|
2999
|
+
const db = getDb();
|
|
3000
|
+
const rows = await db.prepare(`SELECT ip_address, country, user_agent, reason, score
|
|
3001
|
+
FROM banned
|
|
3002
|
+
WHERE ip_address IS NOT NULL`).all();
|
|
3003
|
+
if (rows.length === 0) {
|
|
3004
|
+
log.info("No banned IPs — skipping banned.mmdb");
|
|
3005
|
+
return;
|
|
3006
|
+
}
|
|
3007
|
+
const data = rows.map((r) => ({
|
|
3008
|
+
range: r.ip_address,
|
|
3009
|
+
score: r.score,
|
|
3010
|
+
country: r.country,
|
|
3011
|
+
userAgent: r.user_agent,
|
|
3012
|
+
reason: r.reason,
|
|
3013
|
+
comment: "Automatically generated by @riavzon/botDetector"
|
|
3014
|
+
}));
|
|
3015
|
+
await compiler({
|
|
3016
|
+
type: "mmdb",
|
|
3017
|
+
input: {
|
|
3018
|
+
data,
|
|
3019
|
+
dataBaseName: "banned",
|
|
3020
|
+
outputPath: MMDB_DIR,
|
|
3021
|
+
mmdbPath: mmdbctlPath,
|
|
3022
|
+
generateTypes
|
|
3023
|
+
}
|
|
3024
|
+
});
|
|
3025
|
+
log.info(`banned.mmdb compiled — ${String(data.length)} entries`);
|
|
3026
|
+
if (getConfiguration().generator.deleteAfterBuild) {
|
|
3027
|
+
const compiled = rows.map((r) => r.ip_address);
|
|
3028
|
+
await prep(db, `DELETE FROM banned WHERE ip_address IN (${placeholders(db, compiled.length)})`).run(...compiled);
|
|
3029
|
+
log.info(`Deleted ${String(compiled.length)} rows from banned after build`);
|
|
3030
|
+
}
|
|
3031
|
+
}
|
|
3032
|
+
async function buildHighRiskMmdb(scoreThreshold, generateTypes, mmdbctlPath) {
|
|
3033
|
+
const log = getLogger().child({
|
|
3034
|
+
service: "BOT DETECTOR",
|
|
3035
|
+
branch: "generator",
|
|
3036
|
+
db: "highRisk"
|
|
3037
|
+
});
|
|
3038
|
+
const db = getDb();
|
|
3039
|
+
const rows = await prep(db, `SELECT
|
|
3040
|
+
ip_address, visitor_id, suspicious_activity_score,
|
|
3041
|
+
country, region, region_name, city, lat, lon, timezone,
|
|
3042
|
+
isp, org, browser, browserType, browserVersion, os,
|
|
3043
|
+
device_type, deviceVendor, deviceModel,
|
|
3044
|
+
proxy, hosting, request_count, first_seen, last_seen
|
|
3045
|
+
FROM visitors
|
|
3046
|
+
WHERE suspicious_activity_score >= ? AND ip_address IS NOT NULL`).all(scoreThreshold);
|
|
3047
|
+
if (rows.length === 0) {
|
|
3048
|
+
log.info(`No high-risk visitors (score >= ${String(scoreThreshold)}) — skipping highRisk.mmdb`);
|
|
3049
|
+
return;
|
|
3050
|
+
}
|
|
3051
|
+
const data = rows.map((r) => ({
|
|
3052
|
+
range: r.ip_address,
|
|
3053
|
+
visitorId: r.visitor_id,
|
|
3054
|
+
score: r.suspicious_activity_score,
|
|
3055
|
+
country: r.country,
|
|
3056
|
+
region: r.region,
|
|
3057
|
+
regionName: r.region_name,
|
|
3058
|
+
city: r.city,
|
|
3059
|
+
lat: r.lat,
|
|
3060
|
+
lon: r.lon,
|
|
3061
|
+
timezone: r.timezone,
|
|
3062
|
+
isp: r.isp,
|
|
3063
|
+
org: r.org,
|
|
3064
|
+
browser: r.browser,
|
|
3065
|
+
browserType: r.browserType,
|
|
3066
|
+
browserVersion: r.browserVersion,
|
|
3067
|
+
os: r.os,
|
|
3068
|
+
deviceType: r.device_type,
|
|
3069
|
+
deviceVendor: r.deviceVendor,
|
|
3070
|
+
deviceModel: r.deviceModel,
|
|
3071
|
+
proxy: Boolean(r.proxy),
|
|
3072
|
+
hosting: Boolean(r.hosting),
|
|
3073
|
+
requestCount: r.request_count,
|
|
3074
|
+
firstSeen: r.first_seen ? r.first_seen.toISOString() : null,
|
|
3075
|
+
lastSeen: r.last_seen ? r.last_seen.toISOString() : null,
|
|
3076
|
+
comment: "Automatically generated by @riavzon/botDetector"
|
|
3077
|
+
}));
|
|
3078
|
+
await compiler({
|
|
3079
|
+
type: "mmdb",
|
|
3080
|
+
input: {
|
|
3081
|
+
data,
|
|
3082
|
+
dataBaseName: "highRisk",
|
|
3083
|
+
outputPath: MMDB_DIR,
|
|
3084
|
+
mmdbPath: mmdbctlPath,
|
|
3085
|
+
generateTypes
|
|
3086
|
+
}
|
|
3087
|
+
});
|
|
3088
|
+
log.info(`highRisk.mmdb compiled — ${String(data.length)} entries`);
|
|
3089
|
+
if (getConfiguration().generator.deleteAfterBuild) {
|
|
3090
|
+
const compiled = rows.map((r) => r.ip_address);
|
|
3091
|
+
await prep(db, `DELETE FROM visitors WHERE ip_address IN (${placeholders(db, compiled.length)}) AND suspicious_activity_score >= ${placeholders(db, 1, compiled.length)}`).run(...compiled, scoreThreshold);
|
|
3092
|
+
log.info(`Deleted ${String(compiled.length)} rows from visitors after build`);
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
async function runCycle() {
|
|
3096
|
+
const log = getLogger().child({
|
|
3097
|
+
service: "BOT DETECTOR",
|
|
3098
|
+
branch: "generator"
|
|
3099
|
+
});
|
|
3100
|
+
const { generator } = getConfiguration();
|
|
3101
|
+
log.info("MMDB generation cycle started");
|
|
3102
|
+
try {
|
|
3103
|
+
await Promise.all([buildBannedMmdb(generator.generateTypes, generator.mmdbctlPath), buildHighRiskMmdb(generator.scoreThreshold, generator.generateTypes, generator.mmdbctlPath)]);
|
|
3104
|
+
log.info("MMDB generation cycle complete");
|
|
3105
|
+
} catch (err) {
|
|
3106
|
+
log.error({ err }, "MMDB generation cycle failed");
|
|
3107
|
+
}
|
|
3108
|
+
}
|
|
3109
|
+
async function runGeneration() {
|
|
3110
|
+
return runCycle();
|
|
3111
|
+
}
|
|
3112
|
+
|
|
3113
|
+
//#endregion
|
|
3114
|
+
//#region src/botDetector/db/warmUp.ts
|
|
3115
|
+
async function warmUp() {
|
|
3116
|
+
const db = getDb();
|
|
3117
|
+
await Promise.all(Array.from({ length: 10 }, () => db.sql`SELECT 1`));
|
|
3118
|
+
await prep(db, `SELECT last_seen, request_count
|
|
3119
|
+
FROM visitors
|
|
3120
|
+
WHERE canary_id = ?
|
|
3121
|
+
LIMIT 1`).get("00000000‑warm‑up‑row");
|
|
3122
|
+
consola.info("botDetector is ready!");
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
//#endregion
|
|
3126
|
+
//#region src/botDetector/db/customUpdate.ts
|
|
3127
|
+
async function updateVisitors(data, cookie, visitor_id) {
|
|
3128
|
+
const db = getDb();
|
|
3129
|
+
const log = getLogger().child({
|
|
3130
|
+
service: "BOT DETECTOR",
|
|
3131
|
+
branch: "customUpdate"
|
|
3132
|
+
});
|
|
3133
|
+
const params = [
|
|
3134
|
+
data.userAgent,
|
|
3135
|
+
data.ipAddress,
|
|
3136
|
+
data.country,
|
|
3137
|
+
data.region,
|
|
3138
|
+
data.regionName,
|
|
3139
|
+
data.city,
|
|
3140
|
+
data.district,
|
|
3141
|
+
data.lat,
|
|
3142
|
+
data.lon,
|
|
3143
|
+
data.timezone,
|
|
3144
|
+
data.currency,
|
|
3145
|
+
data.isp,
|
|
3146
|
+
data.org,
|
|
3147
|
+
data.as,
|
|
3148
|
+
data.device_type,
|
|
3149
|
+
data.browser,
|
|
3150
|
+
data.proxy,
|
|
3151
|
+
data.hosting,
|
|
3152
|
+
data.deviceVendor,
|
|
3153
|
+
data.deviceModel,
|
|
3154
|
+
data.browserType,
|
|
3155
|
+
data.browserVersion,
|
|
3156
|
+
data.os
|
|
3157
|
+
];
|
|
3158
|
+
try {
|
|
3159
|
+
if (!(await prep(db, `
|
|
3160
|
+
UPDATE visitors
|
|
3161
|
+
SET
|
|
3162
|
+
user_agent = ?,
|
|
3163
|
+
ip_address = ?,
|
|
3164
|
+
country = ?,
|
|
3165
|
+
region = ?,
|
|
3166
|
+
region_name = ?,
|
|
3167
|
+
city = ?,
|
|
3168
|
+
district = ?,
|
|
3169
|
+
lat = ?,
|
|
3170
|
+
lon = ?,
|
|
3171
|
+
timezone = ?,
|
|
3172
|
+
currency = ?,
|
|
3173
|
+
isp = ?,
|
|
3174
|
+
org = ?,
|
|
3175
|
+
as_org = ?,
|
|
3176
|
+
device_type = ?,
|
|
3177
|
+
browser = ?,
|
|
3178
|
+
proxy = ?,
|
|
3179
|
+
hosting = ?,
|
|
3180
|
+
deviceVendor = ?,
|
|
3181
|
+
deviceModel = ?,
|
|
3182
|
+
browserType = ?,
|
|
3183
|
+
browserVersion = ?,
|
|
3184
|
+
os = ?
|
|
3185
|
+
WHERE canary_id = ?
|
|
3186
|
+
AND visitor_id = ?
|
|
3187
|
+
`).run(...params, cookie, visitor_id)).success) {
|
|
3188
|
+
log.warn({
|
|
3189
|
+
cookie,
|
|
3190
|
+
visitor_id
|
|
3191
|
+
}, `updateVisitors: no rows affected`);
|
|
3192
|
+
return {
|
|
3193
|
+
success: false,
|
|
3194
|
+
reason: `No visitor found for canary_id=${cookie} visitor_id=${visitor_id}`
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
log.info({ visitor_id }, `updateVisitors: visitor updated`);
|
|
3198
|
+
return { success: true };
|
|
3199
|
+
} catch (err) {
|
|
3200
|
+
log.error({ err }, `updateVisitors: query failed`);
|
|
3201
|
+
return {
|
|
3202
|
+
success: false,
|
|
3203
|
+
reason: "DB error — check logs for details"
|
|
3204
|
+
};
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
|
|
3208
|
+
//#endregion
|
|
3209
|
+
export { router as ApiResponse, BadBotDetected, CheckerRegistry, GoodBotDetected, banIp, configuration as defineConfiguration, validator as detectBots, getBatchQueue, getDataSources, getData as getGeoData, getStorage, parseUA, runGeneration, updateBannedIP, updateIsBot, updateVisitors, warmUp };
|
|
3210
|
+
//# sourceMappingURL=main.mjs.map
|