datacenter-ip-ranges 0.0.0-stage → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +432 -2
  3. package/bin/check.js +98 -0
  4. package/data/SOURCES.md +36 -0
  5. package/data/all.txt +12322 -0
  6. package/data/categories/cdn.txt +46 -0
  7. package/data/categories/cloud.txt +6080 -0
  8. package/data/categories/crawler.txt +1303 -0
  9. package/data/categories/privacy-relay.txt +13994 -0
  10. package/data/categories/services.txt +6967 -0
  11. package/data/categories/tor.txt +697 -0
  12. package/data/firewall/datacenter.ipset +12327 -0
  13. package/data/firewall/datacenter.nft +12340 -0
  14. package/data/firewall/tor.ipset +702 -0
  15. package/data/firewall/tor.nft +713 -0
  16. package/data/ipv4.txt +7272 -0
  17. package/data/ipv6.txt +5050 -0
  18. package/data/providers/ahrefsbot.txt +81 -0
  19. package/data/providers/applebot.txt +12 -0
  20. package/data/providers/aws.txt +4114 -0
  21. package/data/providers/bingbot.txt +29 -0
  22. package/data/providers/ccbot.txt +5 -0
  23. package/data/providers/chatgpt-user.txt +225 -0
  24. package/data/providers/cloudflare.txt +22 -0
  25. package/data/providers/digitalocean.txt +241 -0
  26. package/data/providers/duckduckbot.txt +484 -0
  27. package/data/providers/fastly.txt +24 -0
  28. package/data/providers/github.txt +6572 -0
  29. package/data/providers/google-cloud.txt +495 -0
  30. package/data/providers/google-services.txt +395 -0
  31. package/data/providers/google-special-crawlers.txt +36 -0
  32. package/data/providers/google-user-fetchers-google.txt +87 -0
  33. package/data/providers/google-user-fetchers.txt +233 -0
  34. package/data/providers/googlebot.txt +59 -0
  35. package/data/providers/gptbot.txt +15 -0
  36. package/data/providers/icloud-private-relay.txt +13994 -0
  37. package/data/providers/linode.txt +279 -0
  38. package/data/providers/mistralai-user.txt +4 -0
  39. package/data/providers/oai-searchbot.txt +36 -0
  40. package/data/providers/oracle-cloud.txt +809 -0
  41. package/data/providers/perplexity-user.txt +3 -0
  42. package/data/providers/perplexitybot.txt +6 -0
  43. package/data/providers/tor.txt +697 -0
  44. package/data/providers/vultr.txt +142 -0
  45. package/data/ranges.csv +29100 -0
  46. package/data/ranges.json +29101 -0
  47. package/data/sources.json +379 -0
  48. package/package.json +61 -4
  49. package/php/DatacenterIp.php +320 -0
  50. package/src/bots.js +53 -0
  51. package/src/build.js +321 -0
  52. package/src/index.d.ts +55 -0
  53. package/src/index.js +81 -0
  54. package/src/ip.js +215 -0
  55. package/src/matcher.js +146 -0
  56. package/src/sources.js +221 -0
package/src/matcher.js ADDED
@@ -0,0 +1,146 @@
1
+ 'use strict';
2
+
3
+ const { parseAddress, parseCidr, mergeRanges } = require('./ip');
4
+
5
+ // A matcher answers "which lists contain this address" with one binary search.
6
+ // The ranges of all lists are cut into disjoint segments, and each segment
7
+ // carries a bit mask of the lists that cover it. IPv4 uses plain numbers,
8
+ // IPv6 uses BigInt.
9
+
10
+ const MAX_LISTS = 30;
11
+
12
+ function buildFamily(perList, family) {
13
+ const events = [];
14
+ perList.forEach((ranges, idx) => {
15
+ for (const [s, e] of ranges) {
16
+ events.push([s, idx, 1]);
17
+ events.push([e + 1n, idx, -1]);
18
+ }
19
+ });
20
+ events.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
21
+ const counts = new Array(perList.length).fill(0);
22
+ const starts = [];
23
+ const ends = [];
24
+ const masks = [];
25
+ let mask = 0;
26
+ for (let i = 0; i < events.length; ) {
27
+ const pos = events[i][0];
28
+ while (i < events.length && events[i][0] === pos) {
29
+ const [, idx, d] = events[i];
30
+ counts[idx] += d;
31
+ i++;
32
+ }
33
+ mask = 0;
34
+ for (let k = 0; k < counts.length; k++) if (counts[k] > 0) mask |= 1 << k;
35
+ if (i < events.length && mask !== 0) {
36
+ const end = events[i][0] - 1n;
37
+ const last = starts.length - 1;
38
+ if (last >= 0 && masks[last] === mask && ends[last] + 1n === pos) {
39
+ ends[last] = end;
40
+ } else {
41
+ starts.push(pos);
42
+ ends.push(end);
43
+ masks.push(mask);
44
+ }
45
+ }
46
+ }
47
+ if (family === 4) {
48
+ return {
49
+ starts: Float64Array.from(starts, Number),
50
+ ends: Float64Array.from(ends, Number),
51
+ masks: Int32Array.from(masks),
52
+ };
53
+ }
54
+ return { starts, ends, masks: Int32Array.from(masks) };
55
+ }
56
+
57
+ // Fast path for the common case: a plain dotted IPv4 string. Returns -1 if
58
+ // the input is anything else, so the caller can fall back to the full parser.
59
+ function fastIPv4(s) {
60
+ const len = s.length;
61
+ if (len < 7 || len > 15) return -1;
62
+ let n = 0;
63
+ let part = 0;
64
+ let digits = 0;
65
+ let dots = 0;
66
+ for (let i = 0; i < len; i++) {
67
+ const c = s.charCodeAt(i);
68
+ if (c >= 48 && c <= 57) {
69
+ if (digits === 1 && part === 0) return -1;
70
+ part = part * 10 + (c - 48);
71
+ if (++digits > 3 || part > 255) return -1;
72
+ } else if (c === 46) {
73
+ if (digits === 0 || ++dots > 3) return -1;
74
+ n = n * 256 + part;
75
+ part = 0;
76
+ digits = 0;
77
+ } else {
78
+ return -1;
79
+ }
80
+ }
81
+ if (dots !== 3 || digits === 0) return -1;
82
+ return n * 256 + part;
83
+ }
84
+
85
+ function search(table, x) {
86
+ const { starts, ends, masks } = table;
87
+ let lo = 0;
88
+ let hi = starts.length - 1;
89
+ let hit = -1;
90
+ while (lo <= hi) {
91
+ const mid = (lo + hi) >>> 1;
92
+ if (starts[mid] <= x) {
93
+ hit = mid;
94
+ lo = mid + 1;
95
+ } else {
96
+ hi = mid - 1;
97
+ }
98
+ }
99
+ return hit !== -1 && x <= ends[hit] ? masks[hit] : 0;
100
+ }
101
+
102
+ // lists: [{ id, category, cidrs: [string] }]
103
+ function createMatcher(lists) {
104
+ if (lists.length > MAX_LISTS) throw new Error(`at most ${MAX_LISTS} lists per matcher`);
105
+ const v4 = lists.map(() => []);
106
+ const v6 = lists.map(() => []);
107
+ lists.forEach((list, idx) => {
108
+ for (const c of list.cidrs) {
109
+ const r = parseCidr(c);
110
+ if (!r) throw new Error(`invalid CIDR in ${list.id}: ${c}`);
111
+ (r.family === 4 ? v4 : v6)[idx].push([r.start, r.end]);
112
+ }
113
+ });
114
+ const t4 = buildFamily(v4.map(mergeRanges), 4);
115
+ const t6 = buildFamily(v6.map(mergeRanges), 6);
116
+ const meta = lists.map((l) => ({ provider: l.id, category: l.category }));
117
+
118
+ function maskOf(ip) {
119
+ if (typeof ip === 'string') {
120
+ const n = fastIPv4(ip);
121
+ if (n !== -1) return search(t4, n);
122
+ }
123
+ const a = parseAddress(ip);
124
+ if (!a) return -1;
125
+ return a.family === 4 ? search(t4, Number(a.value)) : search(t6, a.value);
126
+ }
127
+
128
+ return {
129
+ size: { ipv4Segments: t4.starts.length, ipv6Segments: t6.starts.length },
130
+ // Returns true if the address is in any list. Invalid input returns false.
131
+ contains(ip) {
132
+ return maskOf(ip) > 0;
133
+ },
134
+ // Returns [{ provider, category }] for every list that contains the address,
135
+ // an empty array if none does, or null if the input is not an IP address.
136
+ lookup(ip) {
137
+ const mask = maskOf(ip);
138
+ if (mask < 0) return null;
139
+ const out = [];
140
+ for (let k = 0; k < meta.length; k++) if (mask & (1 << k)) out.push(meta[k]);
141
+ return out;
142
+ },
143
+ };
144
+ }
145
+
146
+ module.exports = { createMatcher };
package/src/sources.js ADDED
@@ -0,0 +1,221 @@
1
+ 'use strict';
2
+
3
+ // Every feed this project reads. Each one is published by the provider itself.
4
+ // parse() receives the response body and returns a list of CIDR strings.
5
+
6
+ const CATEGORIES = {
7
+ cloud: 'Cloud and VPS hosting',
8
+ cdn: 'CDN and edge networks',
9
+ services: 'Provider services and automation',
10
+ 'privacy-relay': 'Privacy relay egress',
11
+ tor: 'Tor exit nodes',
12
+ crawler: 'Search engine and AI crawlers',
13
+ };
14
+
15
+ // Categories that count as datacenter traffic for isDatacenterIp() and data/all.txt.
16
+ const DATACENTER_CATEGORIES = ['cloud', 'cdn', 'services'];
17
+
18
+ function lines(text) {
19
+ return text
20
+ .split(/\r?\n/)
21
+ .map((l) => l.trim())
22
+ .filter((l) => l !== '' && !l.startsWith('#'));
23
+ }
24
+
25
+ // RFC 8805 geofeed: the first CSV column is the prefix.
26
+ function geofeed(text) {
27
+ return lines(text).map((l) => l.split(',')[0].trim());
28
+ }
29
+
30
+ function json(text) {
31
+ return JSON.parse(text);
32
+ }
33
+
34
+ // The { prefixes: [{ ipv4Prefix } | { ipv6Prefix }] } format that Google
35
+ // introduced and most crawler operators copied.
36
+ function prefixes(text) {
37
+ return json(text).prefixes.map((p) => p.ipv4Prefix || p.ipv6Prefix);
38
+ }
39
+
40
+ function crawler(id, name, url, docs, owner) {
41
+ return {
42
+ id,
43
+ name,
44
+ category: 'crawler',
45
+ url,
46
+ docs,
47
+ terms: `Published by ${owner} so that site operators can verify its crawler. No license is stated.`,
48
+ parse: prefixes,
49
+ };
50
+ }
51
+
52
+ const SOURCES = [
53
+ {
54
+ id: 'aws',
55
+ name: 'Amazon Web Services',
56
+ category: 'cloud',
57
+ url: 'https://ip-ranges.amazonaws.com/ip-ranges.json',
58
+ docs: 'https://docs.aws.amazon.com/vpc/latest/userguide/aws-ip-ranges.html',
59
+ terms: 'Published by AWS for allow lists and filtering. No license is stated.',
60
+ parse(text) {
61
+ const j = json(text);
62
+ return [
63
+ ...j.prefixes.map((p) => p.ip_prefix),
64
+ ...j.ipv6_prefixes.map((p) => p.ipv6_prefix),
65
+ ];
66
+ },
67
+ },
68
+ {
69
+ id: 'google-cloud',
70
+ name: 'Google Cloud',
71
+ category: 'cloud',
72
+ url: 'https://www.gstatic.com/ipranges/cloud.json',
73
+ docs: 'https://support.google.com/a/answer/10026322',
74
+ terms: 'Published by Google for allow lists and filtering. No license is stated.',
75
+ parse: prefixes,
76
+ },
77
+ {
78
+ id: 'google-services',
79
+ name: 'Google services (excluding Google Cloud)',
80
+ category: 'services',
81
+ url: 'https://www.gstatic.com/ipranges/goog.json',
82
+ docs: 'https://support.google.com/a/answer/10026322',
83
+ terms: 'Published by Google for allow lists and filtering. No license is stated.',
84
+ // Google documents goog.json minus cloud.json as the ranges of its own services.
85
+ subtract: 'google-cloud',
86
+ parse: prefixes,
87
+ },
88
+ {
89
+ id: 'oracle-cloud',
90
+ name: 'Oracle Cloud Infrastructure',
91
+ category: 'cloud',
92
+ url: 'https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json',
93
+ docs: 'https://docs.oracle.com/en-us/iaas/Content/General/Concepts/addressranges.htm',
94
+ terms: 'Published by Oracle in its public documentation. No license is stated.',
95
+ parse(text) {
96
+ return json(text).regions.flatMap((r) => r.cidrs.map((c) => c.cidr));
97
+ },
98
+ },
99
+ {
100
+ id: 'digitalocean',
101
+ name: 'DigitalOcean',
102
+ category: 'cloud',
103
+ url: 'https://www.digitalocean.com/geo/google.csv',
104
+ docs: 'https://docs.digitalocean.com/platform/',
105
+ terms: 'Self-published RFC 8805 geofeed, linked from the DigitalOcean docs. No license is stated.',
106
+ parse: geofeed,
107
+ },
108
+ {
109
+ id: 'linode',
110
+ name: 'Akamai Cloud (Linode)',
111
+ category: 'cloud',
112
+ url: 'https://ipgeo.akamai.com/linode-geofeed.csv',
113
+ docs: 'https://rdap.arin.net/registry/ip/172.104.0.1',
114
+ terms: 'Self-published RFC 8805 geofeed, referenced in the Linode network records at ARIN. No license is stated.',
115
+ parse: geofeed,
116
+ },
117
+ {
118
+ id: 'vultr',
119
+ name: 'Vultr',
120
+ category: 'cloud',
121
+ url: 'https://geofeed.constant.com/',
122
+ docs: 'https://rdap.arin.net/registry/ip/45.32.0.1',
123
+ terms: 'Self-published RFC 8805 geofeed of AS20473, referenced in the Vultr network records at ARIN. No license is stated.',
124
+ parse: geofeed,
125
+ },
126
+ {
127
+ id: 'azure',
128
+ name: 'Microsoft Azure',
129
+ category: 'cloud',
130
+ url: 'https://www.microsoft.com/en-us/download/details.aspx?id=56519',
131
+ docs: 'https://learn.microsoft.com/en-us/azure/virtual-network/service-tags-overview',
132
+ terms: 'Microsoft Download Center file. The Microsoft Terms of Use restrict reposting downloads, so this data is not committed. Fetch it locally with --include azure.',
133
+ optIn: true,
134
+ // The weekly file has a new name every week. The download page links the current one.
135
+ async fetchText(get) {
136
+ const page = await get(this.url);
137
+ const m = page.match(/https:\/\/download\.microsoft\.com\/download\/[^"'\s]+\/ServiceTags_Public_\d{8}\.json/i);
138
+ if (!m) throw new Error('no ServiceTags_Public link on the download page');
139
+ return { text: await get(m[0]), resolvedUrl: m[0] };
140
+ },
141
+ parse(text) {
142
+ const tag = json(text).values.find((v) => v.name === 'AzureCloud');
143
+ if (!tag) throw new Error('AzureCloud tag not found');
144
+ return tag.properties.addressPrefixes;
145
+ },
146
+ },
147
+ {
148
+ id: 'cloudflare',
149
+ name: 'Cloudflare',
150
+ category: 'cdn',
151
+ url: 'https://www.cloudflare.com/ips-v4',
152
+ urlV6: 'https://www.cloudflare.com/ips-v6',
153
+ docs: 'https://www.cloudflare.com/ips/',
154
+ terms: 'Published by Cloudflare for allow lists. No license is stated.',
155
+ parse: lines,
156
+ },
157
+ {
158
+ id: 'fastly',
159
+ name: 'Fastly',
160
+ category: 'cdn',
161
+ url: 'https://api.fastly.com/public-ip-list',
162
+ docs: 'https://www.fastly.com/documentation/reference/api/utils/public-ip-list/',
163
+ terms: 'Public, unauthenticated Fastly API endpoint. No license is stated.',
164
+ parse(text) {
165
+ const j = json(text);
166
+ return [...(j.addresses || []), ...(j.ipv6_addresses || [])];
167
+ },
168
+ },
169
+ {
170
+ id: 'github',
171
+ name: 'GitHub (incl. Actions runners)',
172
+ category: 'services',
173
+ url: 'https://api.github.com/meta',
174
+ docs: 'https://docs.github.com/en/rest/meta/meta',
175
+ terms: 'GitHub REST API, covered by the GitHub Terms of Service (API Terms).',
176
+ parse(text) {
177
+ const j = json(text);
178
+ const out = [];
179
+ for (const [key, value] of Object.entries(j)) {
180
+ if (!Array.isArray(value) || key === 'ssh_keys' || key === 'commit_signing_keys') continue;
181
+ for (const v of value) if (typeof v === 'string' && /^[0-9a-fA-F:.]+\/\d+$/.test(v)) out.push(v);
182
+ }
183
+ return out;
184
+ },
185
+ },
186
+ {
187
+ id: 'icloud-private-relay',
188
+ name: 'iCloud Private Relay egress',
189
+ category: 'privacy-relay',
190
+ url: 'https://mask-api.icloud.com/egress-ip-ranges.csv',
191
+ docs: 'https://developer.apple.com/support/prepare-your-network-for-icloud-private-relay/',
192
+ terms: 'Published by Apple for geolocation and server operators. No license is stated.',
193
+ parse: geofeed,
194
+ },
195
+ {
196
+ id: 'tor',
197
+ name: 'Tor exit nodes',
198
+ category: 'tor',
199
+ url: 'https://check.torproject.org/torbulkexitlist',
200
+ docs: 'https://metrics.torproject.org/about.html',
201
+ terms: 'Tor Project data, CC0 1.0 (no copyright).',
202
+ parse: lines,
203
+ },
204
+ crawler('googlebot', 'Googlebot and other Google common crawlers', 'https://developers.google.com/static/crawling/ipranges/common-crawlers.json', 'https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests', 'Google'),
205
+ crawler('google-special-crawlers', 'Google special-case crawlers (AdsBot, Mediapartners)', 'https://developers.google.com/static/crawling/ipranges/special-crawlers.json', 'https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests', 'Google'),
206
+ crawler('google-user-fetchers', 'Google user-triggered fetchers', 'https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json', 'https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests', 'Google'),
207
+ crawler('google-user-fetchers-google', 'Google user-triggered fetchers (Google products)', 'https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json', 'https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests', 'Google'),
208
+ crawler('bingbot', 'Bingbot', 'https://www.bing.com/toolbox/bingbot.json', 'https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26', 'Microsoft'),
209
+ crawler('applebot', 'Applebot', 'https://search.developer.apple.com/applebot.json', 'https://support.apple.com/en-us/119829', 'Apple'),
210
+ crawler('duckduckbot', 'DuckDuckBot', 'https://duckduckgo.com/duckduckbot.json', 'https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot', 'DuckDuckGo'),
211
+ crawler('gptbot', 'OpenAI GPTBot', 'https://openai.com/gptbot.json', 'https://developers.openai.com/api/docs/bots', 'OpenAI'),
212
+ crawler('oai-searchbot', 'OpenAI OAI-SearchBot', 'https://openai.com/searchbot.json', 'https://developers.openai.com/api/docs/bots', 'OpenAI'),
213
+ crawler('chatgpt-user', 'OpenAI ChatGPT-User', 'https://openai.com/chatgpt-user.json', 'https://developers.openai.com/api/docs/bots', 'OpenAI'),
214
+ crawler('perplexitybot', 'PerplexityBot', 'https://www.perplexity.ai/perplexitybot.json', 'https://docs.perplexity.ai/docs/resources/perplexity-crawlers', 'Perplexity'),
215
+ crawler('perplexity-user', 'Perplexity-User', 'https://www.perplexity.ai/perplexity-user.json', 'https://docs.perplexity.ai/docs/resources/perplexity-crawlers', 'Perplexity'),
216
+ crawler('mistralai-user', 'MistralAI-User', 'https://mistral.ai/mistralai-user-ips.json', 'https://docs.mistral.ai/robots', 'Mistral AI'),
217
+ crawler('ccbot', 'Common Crawl CCBot', 'https://index.commoncrawl.org/ccbot.json', 'https://commoncrawl.org/ccbot', 'Common Crawl'),
218
+ crawler('ahrefsbot', 'AhrefsBot and AhrefsSiteAudit', 'https://api.ahrefs.com/v3/public/crawler-ip-ranges', 'https://help.ahrefs.com/en/articles/78658-what-is-the-list-of-your-ip-ranges', 'Ahrefs'),
219
+ ];
220
+
221
+ module.exports = { SOURCES, CATEGORIES, DATACENTER_CATEGORIES, lines, geofeed, prefixes };