datacenter-ip-ranges 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +432 -2
- package/bin/check.js +98 -0
- package/data/SOURCES.md +36 -0
- package/data/all.txt +12322 -0
- package/data/categories/cdn.txt +46 -0
- package/data/categories/cloud.txt +6080 -0
- package/data/categories/crawler.txt +1303 -0
- package/data/categories/privacy-relay.txt +13994 -0
- package/data/categories/services.txt +6967 -0
- package/data/categories/tor.txt +697 -0
- package/data/firewall/datacenter.ipset +12327 -0
- package/data/firewall/datacenter.nft +12340 -0
- package/data/firewall/tor.ipset +702 -0
- package/data/firewall/tor.nft +713 -0
- package/data/ipv4.txt +7272 -0
- package/data/ipv6.txt +5050 -0
- package/data/providers/ahrefsbot.txt +81 -0
- package/data/providers/applebot.txt +12 -0
- package/data/providers/aws.txt +4114 -0
- package/data/providers/bingbot.txt +29 -0
- package/data/providers/ccbot.txt +5 -0
- package/data/providers/chatgpt-user.txt +225 -0
- package/data/providers/cloudflare.txt +22 -0
- package/data/providers/digitalocean.txt +241 -0
- package/data/providers/duckduckbot.txt +484 -0
- package/data/providers/fastly.txt +24 -0
- package/data/providers/github.txt +6572 -0
- package/data/providers/google-cloud.txt +495 -0
- package/data/providers/google-services.txt +395 -0
- package/data/providers/google-special-crawlers.txt +36 -0
- package/data/providers/google-user-fetchers-google.txt +87 -0
- package/data/providers/google-user-fetchers.txt +233 -0
- package/data/providers/googlebot.txt +59 -0
- package/data/providers/gptbot.txt +15 -0
- package/data/providers/icloud-private-relay.txt +13994 -0
- package/data/providers/linode.txt +279 -0
- package/data/providers/mistralai-user.txt +4 -0
- package/data/providers/oai-searchbot.txt +36 -0
- package/data/providers/oracle-cloud.txt +809 -0
- package/data/providers/perplexity-user.txt +3 -0
- package/data/providers/perplexitybot.txt +6 -0
- package/data/providers/tor.txt +697 -0
- package/data/providers/vultr.txt +142 -0
- package/data/ranges.csv +29100 -0
- package/data/ranges.json +29101 -0
- package/data/sources.json +379 -0
- package/package.json +61 -4
- package/php/DatacenterIp.php +320 -0
- package/src/bots.js +53 -0
- package/src/build.js +321 -0
- package/src/index.d.ts +55 -0
- package/src/index.js +81 -0
- package/src/ip.js +215 -0
- package/src/matcher.js +146 -0
- package/src/sources.js +221 -0
package/src/build.js
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Fetches every feed in sources.js, normalizes and merges the ranges, and
|
|
4
|
+
// writes the files in data/.
|
|
5
|
+
//
|
|
6
|
+
// A feed that fails, or that suddenly shrinks by more than half, keeps its
|
|
7
|
+
// previous data and is marked "stale" in data/sources.json. buildData() throws
|
|
8
|
+
// only if every feed failed, and then writes nothing.
|
|
9
|
+
|
|
10
|
+
const fs = require('node:fs');
|
|
11
|
+
const path = require('node:path');
|
|
12
|
+
const ip = require('./ip');
|
|
13
|
+
const { SOURCES, CATEGORIES, DATACENTER_CATEGORIES } = require('./sources');
|
|
14
|
+
|
|
15
|
+
const USER_AGENT = 'datacenter-ip-ranges/1.0 (+https://github.com/moonetix/datacenter-ip-ranges)';
|
|
16
|
+
const TIMEOUT_MS = 60000;
|
|
17
|
+
const SHRINK_LIMIT = 0.5;
|
|
18
|
+
|
|
19
|
+
// Never let a feed mark private, loopback, link-local or other special ranges.
|
|
20
|
+
const SPECIAL = {
|
|
21
|
+
4: ['0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '169.254.0.0/16', '172.16.0.0/12',
|
|
22
|
+
'192.0.0.0/24', '192.0.2.0/24', '192.168.0.0/16', '198.18.0.0/15', '198.51.100.0/24',
|
|
23
|
+
'203.0.113.0/24', '224.0.0.0/4', '240.0.0.0/4'],
|
|
24
|
+
6: ['::/8', '100::/64', '2001:db8::/32', 'fc00::/7', 'fe80::/10', 'ff00::/8'],
|
|
25
|
+
};
|
|
26
|
+
// Anything broader than this is treated as a feed error.
|
|
27
|
+
const MIN_PREFIX = { 4: 8, 6: 16 };
|
|
28
|
+
|
|
29
|
+
async function get(url) {
|
|
30
|
+
const headers = { 'User-Agent': USER_AGENT, Accept: '*/*' };
|
|
31
|
+
if (url.startsWith('https://api.github.com/') && process.env.GITHUB_TOKEN) {
|
|
32
|
+
headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
|
|
33
|
+
}
|
|
34
|
+
let lastError;
|
|
35
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
36
|
+
try {
|
|
37
|
+
const res = await fetch(url, { headers, redirect: 'follow', signal: AbortSignal.timeout(TIMEOUT_MS) });
|
|
38
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
39
|
+
return await res.text();
|
|
40
|
+
} catch (e) {
|
|
41
|
+
lastError = e;
|
|
42
|
+
if (attempt < 3) await new Promise((r) => setTimeout(r, attempt * 3000));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
throw new Error(`${url}: ${lastError.message}`);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function specialRanges(family) {
|
|
49
|
+
return ip.mergeRanges(SPECIAL[family].map((c) => {
|
|
50
|
+
const r = ip.parseCidr(c);
|
|
51
|
+
return [r.start, r.end];
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Turns raw CIDR strings into merged ranges per family, with counts of what was dropped.
|
|
56
|
+
function normalize(cidrs) {
|
|
57
|
+
const acc = { 4: [], 6: [] };
|
|
58
|
+
let invalid = 0;
|
|
59
|
+
let tooBroad = 0;
|
|
60
|
+
for (const c of cidrs) {
|
|
61
|
+
const r = typeof c === 'string' ? ip.parseCidr(c) : null;
|
|
62
|
+
if (!r) { invalid++; continue; }
|
|
63
|
+
if (r.prefix < MIN_PREFIX[r.family]) { tooBroad++; continue; }
|
|
64
|
+
acc[r.family].push([r.start, r.end]);
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
4: ip.subtractRanges(ip.mergeRanges(acc[4]), specialRanges(4)),
|
|
68
|
+
6: ip.subtractRanges(ip.mergeRanges(acc[6]), specialRanges(6)),
|
|
69
|
+
invalid,
|
|
70
|
+
tooBroad,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function readPrevious(outDir, id) {
|
|
75
|
+
const file = path.join(outDir, 'providers', `${id}.txt`);
|
|
76
|
+
if (!fs.existsSync(file)) return null;
|
|
77
|
+
return normalize(fs.readFileSync(file, 'utf8').split('\n').filter(Boolean));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function count(r) {
|
|
81
|
+
return r[4].length + r[6].length;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function toCidrs(r) {
|
|
85
|
+
return [...ip.rangesToCidrs(4, r[4]), ...ip.rangesToCidrs(6, r[6])];
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function writeFile(file, content) {
|
|
89
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
90
|
+
fs.writeFileSync(file, content);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function writeList(file, cidrs) {
|
|
94
|
+
writeFile(file, cidrs.length ? cidrs.join('\n') + '\n' : '');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function union(parts) {
|
|
98
|
+
return {
|
|
99
|
+
4: ip.mergeRanges(parts.flatMap((p) => p[4])),
|
|
100
|
+
6: ip.mergeRanges(parts.flatMap((p) => p[6])),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// nftables sets, meant to be included inside a table block.
|
|
105
|
+
function nftSets(name, v4, v6) {
|
|
106
|
+
const set = (family, type, cidrs) => [
|
|
107
|
+
`set ${name}_${family} {`,
|
|
108
|
+
`\ttype ${type}`,
|
|
109
|
+
'\tflags interval',
|
|
110
|
+
...(cidrs.length ? ['\telements = {', ...cidrs.map((c, i) => `\t\t${c}${i < cidrs.length - 1 ? ',' : ''}`), '\t}'] : []),
|
|
111
|
+
'}',
|
|
112
|
+
];
|
|
113
|
+
return [
|
|
114
|
+
`# ${name} ranges from https://github.com/moonetix/datacenter-ip-ranges`,
|
|
115
|
+
'# Include inside a table, for example:',
|
|
116
|
+
'# table inet filter {',
|
|
117
|
+
`# include "/etc/nftables.d/${name}.nft"`,
|
|
118
|
+
`# chain input { type filter hook input priority 0; ip saddr @${name}_v4 drop; ip6 saddr @${name}_v6 drop; }`,
|
|
119
|
+
'# }',
|
|
120
|
+
...set('v4', 'ipv4_addr', v4),
|
|
121
|
+
...set('v6', 'ipv6_addr', v6),
|
|
122
|
+
'',
|
|
123
|
+
].join('\n');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ipset restore file: ipset restore -exist < name.ipset
|
|
127
|
+
function ipsetFile(name, v4, v6) {
|
|
128
|
+
const size = (n) => Math.max(65536, 2 ** Math.ceil(Math.log2(n * 2 || 1)));
|
|
129
|
+
return [
|
|
130
|
+
`# ${name} ranges from https://github.com/moonetix/datacenter-ip-ranges`,
|
|
131
|
+
`# Load with: ipset restore -exist < ${name}.ipset`,
|
|
132
|
+
`# Then, for example: iptables -I INPUT -m set --match-set ${name}-v4 src -j DROP`,
|
|
133
|
+
`create ${name}-v4 hash:net family inet maxelem ${size(v4.length)}`,
|
|
134
|
+
`create ${name}-v6 hash:net family inet6 maxelem ${size(v6.length)}`,
|
|
135
|
+
...v4.map((c) => `add ${name}-v4 ${c}`),
|
|
136
|
+
...v6.map((c) => `add ${name}-v6 ${c}`),
|
|
137
|
+
'',
|
|
138
|
+
].join('\n');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function csvField(v) {
|
|
142
|
+
return /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Fetches the feeds and writes every file in outDir. Returns the source summary.
|
|
146
|
+
// options: outDir (required), include (opt-in source IDs), sources, get(url), now, log
|
|
147
|
+
async function buildData(options) {
|
|
148
|
+
const outDir = options.outDir;
|
|
149
|
+
const include = options.include || [];
|
|
150
|
+
const sources = options.sources || SOURCES;
|
|
151
|
+
const fetchText = options.get || get;
|
|
152
|
+
const log = options.log || (() => {});
|
|
153
|
+
const selected = sources.filter((s) => !s.optIn || include.includes(s.id));
|
|
154
|
+
const fetchedAt = (options.now || new Date()).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
|
155
|
+
|
|
156
|
+
// Fetch everything in parallel.
|
|
157
|
+
const raw = await Promise.all(selected.map(async (src) => {
|
|
158
|
+
try {
|
|
159
|
+
let text;
|
|
160
|
+
let resolvedUrl = src.url;
|
|
161
|
+
if (src.fetchText) {
|
|
162
|
+
({ text, resolvedUrl } = await src.fetchText(fetchText));
|
|
163
|
+
} else {
|
|
164
|
+
text = await fetchText(src.url);
|
|
165
|
+
if (src.urlV6) text += '\n' + (await fetchText(src.urlV6));
|
|
166
|
+
}
|
|
167
|
+
return { src, cidrs: src.parse(text), resolvedUrl };
|
|
168
|
+
} catch (e) {
|
|
169
|
+
return { src, error: e.message };
|
|
170
|
+
}
|
|
171
|
+
}));
|
|
172
|
+
|
|
173
|
+
const results = new Map();
|
|
174
|
+
for (const { src, cidrs, resolvedUrl, error } of raw) {
|
|
175
|
+
const previous = readPrevious(outDir, src.id);
|
|
176
|
+
let ranges = null;
|
|
177
|
+
let status = 'ok';
|
|
178
|
+
let note = '';
|
|
179
|
+
if (error) {
|
|
180
|
+
status = 'stale';
|
|
181
|
+
note = `fetch or parse failed: ${error}`;
|
|
182
|
+
} else {
|
|
183
|
+
ranges = normalize(cidrs);
|
|
184
|
+
if (ranges.invalid || ranges.tooBroad) {
|
|
185
|
+
note = `dropped ${ranges.invalid} invalid and ${ranges.tooBroad} too broad entries`;
|
|
186
|
+
}
|
|
187
|
+
if (count(ranges) === 0) {
|
|
188
|
+
status = 'stale';
|
|
189
|
+
note = 'feed returned no ranges';
|
|
190
|
+
ranges = null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
results.set(src.id, { src, ranges, previous, status, note, resolvedUrl: resolvedUrl || src.url });
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// google-services is goog.json minus cloud.json.
|
|
197
|
+
for (const r of results.values()) {
|
|
198
|
+
if (!r.src.subtract || !r.ranges) continue;
|
|
199
|
+
const base = results.get(r.src.subtract);
|
|
200
|
+
const minus = (base && (base.ranges || base.previous)) || readPrevious(outDir, r.src.subtract);
|
|
201
|
+
if (!minus) {
|
|
202
|
+
r.status = 'stale';
|
|
203
|
+
r.note = `cannot subtract ${r.src.subtract}: no data`;
|
|
204
|
+
r.ranges = null;
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
r.ranges = { 4: ip.subtractRanges(r.ranges[4], minus[4]), 6: ip.subtractRanges(r.ranges[6], minus[6]) };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Shrink guard: keep the previous list if a feed suddenly loses more than half its ranges.
|
|
211
|
+
for (const r of results.values()) {
|
|
212
|
+
if (r.ranges && r.previous && count(r.previous) > 20 && count(r.ranges) < count(r.previous) * SHRINK_LIMIT) {
|
|
213
|
+
r.status = 'stale';
|
|
214
|
+
r.note = `new list has ${count(r.ranges)} ranges, previous had ${count(r.previous)}; kept previous`;
|
|
215
|
+
r.ranges = null;
|
|
216
|
+
}
|
|
217
|
+
if (!r.ranges) r.ranges = r.previous;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if ([...results.values()].every((r) => r.status !== 'ok')) {
|
|
221
|
+
const why = [...results.values()].map((r) => `${r.src.id}: ${r.note}`).join('\n');
|
|
222
|
+
throw new Error(`Every feed failed. Nothing was written.\n${why}`);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// Per provider files.
|
|
226
|
+
const sourcesMeta = [];
|
|
227
|
+
const rows = [];
|
|
228
|
+
const byCategory = {};
|
|
229
|
+
for (const src of sources) {
|
|
230
|
+
const r = results.get(src.id);
|
|
231
|
+
const entry = {
|
|
232
|
+
id: src.id,
|
|
233
|
+
name: src.name,
|
|
234
|
+
category: src.category,
|
|
235
|
+
url: src.url,
|
|
236
|
+
url_v6: src.urlV6,
|
|
237
|
+
docs: src.docs,
|
|
238
|
+
terms: src.terms,
|
|
239
|
+
};
|
|
240
|
+
if (!r || !r.ranges) {
|
|
241
|
+
entry.status = src.optIn && !results.has(src.id) ? 'not included (opt-in)' : 'unavailable';
|
|
242
|
+
if (r && r.note) entry.note = r.note;
|
|
243
|
+
sourcesMeta.push(entry);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
const cidrs = toCidrs(r.ranges);
|
|
247
|
+
const file = `providers/${src.id}.txt`;
|
|
248
|
+
writeList(path.join(outDir, file), cidrs);
|
|
249
|
+
Object.assign(entry, {
|
|
250
|
+
file,
|
|
251
|
+
status: r.status,
|
|
252
|
+
ipv4: r.ranges[4].length ? ip.rangesToCidrs(4, r.ranges[4]).length : 0,
|
|
253
|
+
ipv6: r.ranges[6].length ? ip.rangesToCidrs(6, r.ranges[6]).length : 0,
|
|
254
|
+
fetched_at: r.status === 'ok' ? fetchedAt : undefined,
|
|
255
|
+
resolved_url: r.resolvedUrl !== src.url ? r.resolvedUrl : undefined,
|
|
256
|
+
note: r.note || undefined,
|
|
257
|
+
});
|
|
258
|
+
sourcesMeta.push(entry);
|
|
259
|
+
(byCategory[src.category] ||= []).push(r.ranges);
|
|
260
|
+
for (const c of cidrs) {
|
|
261
|
+
const source = src.urlV6 && c.includes(':') ? src.urlV6 : r.resolvedUrl;
|
|
262
|
+
rows.push({ provider: src.id, category: src.category, cidr: c, source });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Per category files and the combined datacenter lists.
|
|
267
|
+
for (const cat of Object.keys(CATEGORIES)) {
|
|
268
|
+
const u = union(byCategory[cat] || []);
|
|
269
|
+
writeList(path.join(outDir, 'categories', `${cat}.txt`), toCidrs(u));
|
|
270
|
+
}
|
|
271
|
+
const dc = union(DATACENTER_CATEGORIES.flatMap((c) => byCategory[c] || []));
|
|
272
|
+
const v4 = ip.rangesToCidrs(4, dc[4]);
|
|
273
|
+
const v6 = ip.rangesToCidrs(6, dc[6]);
|
|
274
|
+
writeList(path.join(outDir, 'all.txt'), [...v4, ...v6]);
|
|
275
|
+
writeList(path.join(outDir, 'ipv4.txt'), v4);
|
|
276
|
+
writeList(path.join(outDir, 'ipv6.txt'), v6);
|
|
277
|
+
|
|
278
|
+
// Firewall formats for the datacenter lists and the Tor exits.
|
|
279
|
+
const tor = union(byCategory.tor || []);
|
|
280
|
+
for (const [name, v4list, v6list] of [
|
|
281
|
+
['datacenter', v4, v6],
|
|
282
|
+
['tor', ip.rangesToCidrs(4, tor[4]), ip.rangesToCidrs(6, tor[6])],
|
|
283
|
+
]) {
|
|
284
|
+
writeFile(path.join(outDir, 'firewall', `${name}.nft`), nftSets(name, v4list, v6list));
|
|
285
|
+
writeFile(path.join(outDir, 'firewall', `${name}.ipset`), ipsetFile(name, v4list, v6list));
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// ranges.json and ranges.csv: every provider range, all categories.
|
|
289
|
+
writeFile(path.join(outDir, 'ranges.json'), '[\n' + rows.map((r) => JSON.stringify(r)).join(',\n') + '\n]\n');
|
|
290
|
+
writeFile(
|
|
291
|
+
path.join(outDir, 'ranges.csv'),
|
|
292
|
+
'provider,category,cidr,source\n' + rows.map((r) => [r.provider, r.category, r.cidr, r.source].map(csvField).join(',')).join('\n') + '\n',
|
|
293
|
+
);
|
|
294
|
+
|
|
295
|
+
// sources.json and SOURCES.md: what was fetched, when, and how much.
|
|
296
|
+
const meta = { updated_at: fetchedAt, datacenter_categories: DATACENTER_CATEGORIES, categories: CATEGORIES, sources: sourcesMeta };
|
|
297
|
+
writeFile(path.join(outDir, 'sources.json'), JSON.stringify(meta, null, 2) + '\n');
|
|
298
|
+
|
|
299
|
+
const md = [
|
|
300
|
+
'# Sources',
|
|
301
|
+
'',
|
|
302
|
+
`Last run ${fetchedAt} by \`scripts/update.js\`. This file is rewritten on every run.`,
|
|
303
|
+
'',
|
|
304
|
+
`Datacenter lists (\`all.txt\`, \`ipv4.txt\`, \`ipv6.txt\`): ${v4.length} IPv4 and ${v6.length} IPv6 CIDR blocks.`,
|
|
305
|
+
'',
|
|
306
|
+
'| Provider | Category | IPv4 | IPv6 | Status | Fetched (UTC) | Feed |',
|
|
307
|
+
'| --- | --- | ---: | ---: | --- | --- | --- |',
|
|
308
|
+
...sourcesMeta.map((s) => `| ${s.name} | ${s.category} | ${s.ipv4 ?? '-'} | ${s.ipv6 ?? '-'} | ${s.status}${s.note ? ` (${s.note})` : ''} | ${s.fetched_at || '-'} | ${[s.url, s.url_v6].filter(Boolean).map((u) => `[${new URL(u).host}${new URL(u).pathname}](${u})`).join(', ')} |`),
|
|
309
|
+
'',
|
|
310
|
+
].join('\n');
|
|
311
|
+
writeFile(path.join(outDir, 'SOURCES.md'), md);
|
|
312
|
+
|
|
313
|
+
for (const s of sourcesMeta) {
|
|
314
|
+
const counts = s.file ? `${String(s.ipv4).padStart(7)} v4 ${String(s.ipv6).padStart(7)} v6` : ''.padStart(21);
|
|
315
|
+
log(`${s.id.padEnd(28)} ${counts} ${s.status}${s.note ? ': ' + s.note : ''}`);
|
|
316
|
+
}
|
|
317
|
+
log(`datacenter total ${String(v4.length).padStart(7)} v4 ${String(v6.length).padStart(7)} v6`);
|
|
318
|
+
return { updatedAt: fetchedAt, sources: sourcesMeta, ipv4: v4.length, ipv6: v6.length };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
module.exports = { buildData, normalize, SPECIAL, MIN_PREFIX };
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
export type Category = 'cloud' | 'cdn' | 'services' | 'privacy-relay' | 'tor' | 'crawler';
|
|
2
|
+
|
|
3
|
+
export interface Match {
|
|
4
|
+
provider: string;
|
|
5
|
+
category: Category;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface Matcher {
|
|
9
|
+
size: { ipv4Segments: number; ipv6Segments: number };
|
|
10
|
+
/** True if the address is in any loaded list. Invalid input returns false. */
|
|
11
|
+
contains(ip: string): boolean;
|
|
12
|
+
/** Every loaded list that contains the address, [] if none, null if the input is not an IP address. */
|
|
13
|
+
lookup(ip: string): Match[] | null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface LoadOptions {
|
|
17
|
+
/** Folder with sources.json and providers/. Default: the package's data/. */
|
|
18
|
+
dataDir?: string;
|
|
19
|
+
/** Categories to load. Default: cloud, cdn, services. */
|
|
20
|
+
categories?: Category[];
|
|
21
|
+
/** Provider IDs to load, instead of categories. */
|
|
22
|
+
providers?: string[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface Bot {
|
|
26
|
+
id: string;
|
|
27
|
+
name: string;
|
|
28
|
+
/** User-agent tokens, matched case-insensitively anywhere in the string. */
|
|
29
|
+
tokens: string[];
|
|
30
|
+
/** Provider lists (category "crawler") the bot's requests must come from. */
|
|
31
|
+
providers: string[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface BotCheck {
|
|
35
|
+
bot: string;
|
|
36
|
+
name: string;
|
|
37
|
+
/** True if the address is in the ranges the bot's operator publishes. */
|
|
38
|
+
verified: boolean;
|
|
39
|
+
/** The provider list that matched, or null. */
|
|
40
|
+
provider: string | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** True for cloud, hosting, CDN and provider-service ranges. Tor, Private Relay and crawlers are not included. */
|
|
44
|
+
export function isDatacenterIp(ip: string): boolean;
|
|
45
|
+
/** Every list in every category that contains the address. */
|
|
46
|
+
export function lookup(ip: string): Match[] | null;
|
|
47
|
+
/** null if the user agent claims no known crawler; otherwise whether the address belongs to that crawler. */
|
|
48
|
+
export function verifyBot(userAgent: string, ip: string, options?: { dataDir?: string }): BotCheck | null;
|
|
49
|
+
/** The bot a user-agent string claims to be, or null. */
|
|
50
|
+
export function claimedBot(userAgent: string): Bot | null;
|
|
51
|
+
export function load(options?: LoadOptions): Matcher;
|
|
52
|
+
export function createMatcher(lists: { id: string; category: string; cidrs: string[] }[]): Matcher;
|
|
53
|
+
export function readList(file: string): string[];
|
|
54
|
+
export const BOTS: Bot[];
|
|
55
|
+
export const DEFAULT_DATA_DIR: string;
|
package/src/index.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const { createMatcher } = require('./matcher');
|
|
6
|
+
const { CATEGORIES, DATACENTER_CATEGORIES } = require('./sources');
|
|
7
|
+
const bots = require('./bots');
|
|
8
|
+
|
|
9
|
+
const DEFAULT_DATA_DIR = path.join(__dirname, '..', 'data');
|
|
10
|
+
|
|
11
|
+
function readList(file) {
|
|
12
|
+
return fs
|
|
13
|
+
.readFileSync(file, 'utf8')
|
|
14
|
+
.split('\n')
|
|
15
|
+
.map((l) => l.trim())
|
|
16
|
+
.filter((l) => l !== '' && !l.startsWith('#'));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Builds a matcher from the files in data/.
|
|
20
|
+
// options.dataDir folder with sources.json and providers/ (default: this package's data/)
|
|
21
|
+
// options.categories categories to load (default: cloud, cdn, services)
|
|
22
|
+
// options.providers provider IDs to load (default: every provider in those categories)
|
|
23
|
+
function load(options = {}) {
|
|
24
|
+
const dataDir = options.dataDir || DEFAULT_DATA_DIR;
|
|
25
|
+
const meta = JSON.parse(fs.readFileSync(path.join(dataDir, 'sources.json'), 'utf8'));
|
|
26
|
+
const categories = options.categories || DATACENTER_CATEGORIES;
|
|
27
|
+
const lists = [];
|
|
28
|
+
for (const src of meta.sources) {
|
|
29
|
+
if (!src.file) continue;
|
|
30
|
+
if (options.providers ? !options.providers.includes(src.id) : !categories.includes(src.category)) continue;
|
|
31
|
+
const file = path.join(dataDir, src.file);
|
|
32
|
+
if (!fs.existsSync(file)) continue;
|
|
33
|
+
lists.push({ id: src.id, category: src.category, cidrs: readList(file) });
|
|
34
|
+
}
|
|
35
|
+
return createMatcher(lists);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
let defaultMatcher = null;
|
|
39
|
+
let everythingMatcher = null;
|
|
40
|
+
const crawlerMatchers = new Map();
|
|
41
|
+
|
|
42
|
+
// True if the address belongs to a cloud, hosting, CDN or provider-service range.
|
|
43
|
+
// Tor exits and iCloud Private Relay are not included; use lookup() for those.
|
|
44
|
+
function isDatacenterIp(ip) {
|
|
45
|
+
if (!defaultMatcher) defaultMatcher = load();
|
|
46
|
+
return defaultMatcher.contains(ip);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Every list that contains the address, in all categories:
|
|
50
|
+
// [{ provider, category }], [] when nothing matches, null for invalid input.
|
|
51
|
+
function lookup(ip) {
|
|
52
|
+
if (!everythingMatcher) {
|
|
53
|
+
everythingMatcher = load({ categories: Object.keys(CATEGORIES) });
|
|
54
|
+
}
|
|
55
|
+
return everythingMatcher.lookup(ip);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Checks a request that claims to come from a known crawler (Googlebot, Bingbot,
|
|
59
|
+
// GPTBot, ...) against the ranges that crawler's operator publishes.
|
|
60
|
+
// Returns null if the user agent claims no known crawler, otherwise
|
|
61
|
+
// { bot, name, verified, provider }. verified: false means the claim is false
|
|
62
|
+
// or the address is not one the operator lists.
|
|
63
|
+
// options.dataDir folder with sources.json and providers/ (default: this package's data/),
|
|
64
|
+
// loaded once per folder
|
|
65
|
+
function verifyBot(userAgent, ip, options = {}) {
|
|
66
|
+
const dataDir = options.dataDir || DEFAULT_DATA_DIR;
|
|
67
|
+
if (!crawlerMatchers.has(dataDir)) crawlerMatchers.set(dataDir, load({ dataDir, categories: ['crawler'] }));
|
|
68
|
+
return bots.verify(userAgent, ip, crawlerMatchers.get(dataDir));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
module.exports = {
|
|
72
|
+
isDatacenterIp,
|
|
73
|
+
lookup,
|
|
74
|
+
verifyBot,
|
|
75
|
+
load,
|
|
76
|
+
createMatcher,
|
|
77
|
+
readList,
|
|
78
|
+
BOTS: bots.BOTS,
|
|
79
|
+
claimedBot: bots.claimedBot,
|
|
80
|
+
DEFAULT_DATA_DIR,
|
|
81
|
+
};
|
package/src/ip.js
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// IPv4 and IPv6 parsing, formatting and range arithmetic.
|
|
4
|
+
// Addresses are BigInt values. Ranges are inclusive [start, end] pairs.
|
|
5
|
+
|
|
6
|
+
const V4_MAX = (1n << 32n) - 1n;
|
|
7
|
+
const V6_MAX = (1n << 128n) - 1n;
|
|
8
|
+
const V4_MAPPED_PREFIX = 0xffffn << 32n;
|
|
9
|
+
|
|
10
|
+
function parseIPv4(s) {
|
|
11
|
+
const parts = s.split('.');
|
|
12
|
+
if (parts.length !== 4) return null;
|
|
13
|
+
let n = 0n;
|
|
14
|
+
for (const p of parts) {
|
|
15
|
+
if (!/^(0|[1-9][0-9]{0,2})$/.test(p)) return null;
|
|
16
|
+
const v = Number(p);
|
|
17
|
+
if (v > 255) return null;
|
|
18
|
+
n = (n << 8n) | BigInt(v);
|
|
19
|
+
}
|
|
20
|
+
return n;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function parseIPv6(s) {
|
|
24
|
+
if (!/^[0-9a-fA-F:.]+$/.test(s)) return null;
|
|
25
|
+
let tail = [];
|
|
26
|
+
const lastColon = s.lastIndexOf(':');
|
|
27
|
+
if (s.indexOf('.') !== -1) {
|
|
28
|
+
const v4 = parseIPv4(s.slice(lastColon + 1));
|
|
29
|
+
if (v4 === null) return null;
|
|
30
|
+
tail = [Number(v4 >> 16n), Number(v4 & 0xffffn)];
|
|
31
|
+
s = s.slice(0, lastColon + 1);
|
|
32
|
+
if (!s.endsWith('::')) s = s.slice(0, -1);
|
|
33
|
+
if (s === '') return null;
|
|
34
|
+
}
|
|
35
|
+
const halves = s.split('::');
|
|
36
|
+
if (halves.length > 2) return null;
|
|
37
|
+
const toGroups = (h) => (h === '' ? [] : h.split(':'));
|
|
38
|
+
const head = toGroups(halves[0]);
|
|
39
|
+
const rest = halves.length === 2 ? toGroups(halves[1]) : [];
|
|
40
|
+
for (const g of head.concat(rest)) {
|
|
41
|
+
if (!/^[0-9a-fA-F]{1,4}$/.test(g)) return null;
|
|
42
|
+
}
|
|
43
|
+
const given = head.length + rest.length + tail.length;
|
|
44
|
+
let groups;
|
|
45
|
+
if (halves.length === 2) {
|
|
46
|
+
if (given > 7) return null;
|
|
47
|
+
groups = head.concat(new Array(8 - given).fill('0'), rest);
|
|
48
|
+
} else {
|
|
49
|
+
if (given !== 8) return null;
|
|
50
|
+
groups = head;
|
|
51
|
+
}
|
|
52
|
+
let n = 0n;
|
|
53
|
+
for (const g of groups) n = (n << 16n) | BigInt(parseInt(g, 16));
|
|
54
|
+
for (const t of tail) n = (n << 16n) | BigInt(t);
|
|
55
|
+
return n;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Parses a single address as it may arrive from a socket or a header.
|
|
59
|
+
// Accepts "1.2.3.4", "2001:db8::1", "[2001:db8::1]", "fe80::1%eth0" and
|
|
60
|
+
// IPv4-mapped IPv6 such as "::ffff:1.2.3.4", which is returned as IPv4.
|
|
61
|
+
function parseAddress(input) {
|
|
62
|
+
if (typeof input !== 'string') return null;
|
|
63
|
+
let s = input.trim();
|
|
64
|
+
if (s.startsWith('[') && s.endsWith(']')) s = s.slice(1, -1);
|
|
65
|
+
const zone = s.indexOf('%');
|
|
66
|
+
if (zone !== -1) s = s.slice(0, zone);
|
|
67
|
+
if (s.indexOf(':') === -1) {
|
|
68
|
+
const v4 = parseIPv4(s);
|
|
69
|
+
return v4 === null ? null : { family: 4, value: v4 };
|
|
70
|
+
}
|
|
71
|
+
const v6 = parseIPv6(s);
|
|
72
|
+
if (v6 === null) return null;
|
|
73
|
+
if (v6 >> 32n === 0xffffn) return { family: 4, value: v6 & V4_MAX };
|
|
74
|
+
return { family: 6, value: v6 };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Parses "a.b.c.d/len", "x::/len" or a bare address. Host bits are cleared.
|
|
78
|
+
function parseCidr(input) {
|
|
79
|
+
if (typeof input !== 'string') return null;
|
|
80
|
+
const s = input.trim();
|
|
81
|
+
const slash = s.indexOf('/');
|
|
82
|
+
const addr = slash === -1 ? s : s.slice(0, slash);
|
|
83
|
+
if (addr.indexOf('%') !== -1 || addr.startsWith('[')) return null;
|
|
84
|
+
const family = addr.indexOf(':') === -1 ? 4 : 6;
|
|
85
|
+
const value = family === 4 ? parseIPv4(addr) : parseIPv6(addr);
|
|
86
|
+
if (value === null) return null;
|
|
87
|
+
const bits = family === 4 ? 32 : 128;
|
|
88
|
+
let prefix = bits;
|
|
89
|
+
if (slash !== -1) {
|
|
90
|
+
const p = s.slice(slash + 1);
|
|
91
|
+
if (!/^(0|[1-9][0-9]{0,2})$/.test(p)) return null;
|
|
92
|
+
prefix = Number(p);
|
|
93
|
+
if (prefix > bits) return null;
|
|
94
|
+
}
|
|
95
|
+
const hostBits = BigInt(bits - prefix);
|
|
96
|
+
const start = (value >> hostBits) << hostBits;
|
|
97
|
+
const end = start + (1n << hostBits) - 1n;
|
|
98
|
+
return { family, start, end, prefix };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function formatIPv4(n) {
|
|
102
|
+
return [24n, 16n, 8n, 0n].map((sh) => String((n >> sh) & 0xffn)).join('.');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// RFC 5952 canonical text form.
|
|
106
|
+
function formatIPv6(n) {
|
|
107
|
+
const g = [];
|
|
108
|
+
for (let i = 7; i >= 0; i--) g.push(Number((n >> BigInt(i * 16)) & 0xffffn));
|
|
109
|
+
let bestStart = -1;
|
|
110
|
+
let bestLen = 1;
|
|
111
|
+
for (let i = 0; i < 8; ) {
|
|
112
|
+
if (g[i] !== 0) { i++; continue; }
|
|
113
|
+
let j = i;
|
|
114
|
+
while (j < 8 && g[j] === 0) j++;
|
|
115
|
+
if (j - i > bestLen) { bestStart = i; bestLen = j - i; }
|
|
116
|
+
i = j;
|
|
117
|
+
}
|
|
118
|
+
const hex = g.map((x) => x.toString(16));
|
|
119
|
+
if (bestStart === -1) return hex.join(':');
|
|
120
|
+
const left = hex.slice(0, bestStart).join(':');
|
|
121
|
+
const right = hex.slice(bestStart + bestLen).join(':');
|
|
122
|
+
return left + '::' + right;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function formatAddress(family, n) {
|
|
126
|
+
return family === 4 ? formatIPv4(n) : formatIPv6(n);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Sorts and merges overlapping or adjacent ranges.
|
|
130
|
+
function mergeRanges(ranges) {
|
|
131
|
+
const sorted = ranges.slice().sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
132
|
+
const out = [];
|
|
133
|
+
for (const [s, e] of sorted) {
|
|
134
|
+
const last = out[out.length - 1];
|
|
135
|
+
if (last && s <= last[1] + 1n) {
|
|
136
|
+
if (e > last[1]) last[1] = e;
|
|
137
|
+
} else {
|
|
138
|
+
out.push([s, e]);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Removes every address in b from a. Both inputs must be merged.
|
|
145
|
+
function subtractRanges(a, b) {
|
|
146
|
+
const out = [];
|
|
147
|
+
let j = 0;
|
|
148
|
+
for (const [s0, e] of a) {
|
|
149
|
+
let s = s0;
|
|
150
|
+
while (j < b.length && b[j][1] < s) j++;
|
|
151
|
+
let k = j;
|
|
152
|
+
while (s <= e) {
|
|
153
|
+
if (k >= b.length || b[k][0] > e) { out.push([s, e]); break; }
|
|
154
|
+
const [bs, be] = b[k];
|
|
155
|
+
if (bs > s) out.push([s, bs - 1n]);
|
|
156
|
+
s = be + 1n;
|
|
157
|
+
k++;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function intersects(a, b) {
|
|
164
|
+
let i = 0;
|
|
165
|
+
let j = 0;
|
|
166
|
+
while (i < a.length && j < b.length) {
|
|
167
|
+
if (a[i][1] < b[j][0]) i++;
|
|
168
|
+
else if (b[j][1] < a[i][0]) j++;
|
|
169
|
+
else return true;
|
|
170
|
+
}
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Splits an inclusive range into the smallest list of CIDR blocks.
|
|
175
|
+
function rangeToCidrs(family, start, end) {
|
|
176
|
+
const bits = family === 4 ? 32 : 128;
|
|
177
|
+
const out = [];
|
|
178
|
+
let s = start;
|
|
179
|
+
while (s <= end) {
|
|
180
|
+
let size = 0;
|
|
181
|
+
while (size < bits) {
|
|
182
|
+
const next = size + 1;
|
|
183
|
+
const block = 1n << BigInt(next);
|
|
184
|
+
if (s % block !== 0n || s + block - 1n > end) break;
|
|
185
|
+
size = next;
|
|
186
|
+
}
|
|
187
|
+
out.push(formatAddress(family, s) + '/' + (bits - size));
|
|
188
|
+
s += 1n << BigInt(size);
|
|
189
|
+
}
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function rangesToCidrs(family, ranges) {
|
|
194
|
+
const out = [];
|
|
195
|
+
for (const [s, e] of ranges) out.push(...rangeToCidrs(family, s, e));
|
|
196
|
+
return out;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
module.exports = {
|
|
200
|
+
V4_MAX,
|
|
201
|
+
V6_MAX,
|
|
202
|
+
V4_MAPPED_PREFIX,
|
|
203
|
+
parseIPv4,
|
|
204
|
+
parseIPv6,
|
|
205
|
+
parseAddress,
|
|
206
|
+
parseCidr,
|
|
207
|
+
formatIPv4,
|
|
208
|
+
formatIPv6,
|
|
209
|
+
formatAddress,
|
|
210
|
+
mergeRanges,
|
|
211
|
+
subtractRanges,
|
|
212
|
+
intersects,
|
|
213
|
+
rangeToCidrs,
|
|
214
|
+
rangesToCidrs,
|
|
215
|
+
};
|