@riavzon/shield-base 1.2.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/README.md +188 -130
  2. package/dist/chunk-MZE5OD6T.js +53 -0
  3. package/dist/chunk-MZE5OD6T.js.map +1 -0
  4. package/dist/cli.js +7 -8
  5. package/dist/cli.js.map +1 -1
  6. package/dist/main.d.ts +20 -2
  7. package/dist/main.js +1 -1
  8. package/package.json +2 -1
  9. package/dist/chunk-IXNJSTI7.js +0 -45
  10. package/dist/chunk-IXNJSTI7.js.map +0 -1
package/README.md CHANGED
@@ -2,14 +2,14 @@
2
2
 
3
3
  Shield-Base is a command line tool designed to aggregate, process, and compile network data into offline binary formats.
4
4
 
5
- It fetches data from multiple public sources, including [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) routing tables, geographic location databases, and threat intelligence lists, and consolidates them for use in security analysis and traffic filtering.
5
+ It fetches data from multiple public sources, including [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) routing tables, geographic location databases, threat intelligence lists, and common crawlers ip's and consolidates them for use in security analysis and traffic filtering.
6
6
 
7
7
  The tool can be used as both interactive cli powered by [Consola](https://github.com/unjs/consola/tree/main), and programmatically.
8
8
 
9
9
  ## Features
10
10
 
11
11
  - Comes with an Installation wizard, choose only the databases you need, or compile them all, supports flag based execution for CI/CD environments.
12
- - Merges ASN, GeoIP, Tor exit nodes, and Threat data into a single pipeline.
12
+ - Merges ASN, GeoIP, Tor exit nodes, Threat data, and verified crawler datasets into a single pipeline.
13
13
  - Automatically compiles processed data into .mmdb formats using [mmdbctl](https://github.com/ipinfo/mmdbctl).
14
14
  - Written fully in Typescript
15
15
 
@@ -49,8 +49,48 @@ The tool allows you to choose from multiple security blocklists, categorized int
49
49
  The tool fetches data and ip lists directly from the [Tor Project's Onionoo API](https://metrics.torproject.org/onionoo.html),
50
50
  And a list of known public proxies from different sources, such as [awesome-lists](https://github.com/mthcht/awesome-lists/tree/main/Lists/PROXY), and [FireHOL](https://github.com/firehol/blocklist-ipsets/blob/master/firehol_proxies.netset) then merges them into a single formatted database.
51
51
 
52
- >[!TIP]
53
- > The data sets are updated regularly, for each list, you can use the `-refresh` or `--refreshAll` to refresh your datasets.
52
+ ### 5. Verified Crawlers
53
+ Shield-Base also fetches IP ranges for major search engine crawlers and legitimate automated agents (Google, Bing, Apple, Meta, etc.).
54
+ You can easily customize it to provide additional URLs when used programmatically, or you can use its functions to feed it raw HTML/text to get clean IPs back.
55
+
56
+ Data is extracted directly from official provider geofeeds (JSON/CSV) and HTML pages.
57
+ This fetcher uses a tiered fetch with a fallback mechanism that will use `curl` to bypass anti scraping measures on social media geofeeds when it detects a regular fetch being blocked.
58
+
59
+ > [!WARNING]
60
+ > Make sure `curl` is installed when you use this data source.
61
+
62
+ #### Example Usage
63
+ **Command Line**
64
+
65
+ Simply run `npx shield-base --seo` and the database will be built with around 3400 records.
66
+
67
+ **Programmatic Usage**
68
+
69
+ If you want to provide custom URLs to fetch into the database, you need to define an array of objects including the provider `name`, the list of `urls` to process, and the type of data at the URL, for example:
70
+
71
+ ```ts
72
+ const myCustomCrawlersUrls = [
73
+ {
74
+ name: 'cloudflare', // REQUIRED
75
+
76
+ /** REQUIRED. If the links you are providing include a regular html/markdown/other-raw-text-data page, use HTML. If it is a link to a CSV file, use CSV.
77
+ * If it is a JSON (e.g., https://developers.google.com/static/search/apis/ipranges/googlebot.json), use JSON.
78
+ */
79
+ type: 'CSV' | 'JSON' | 'HTML',
80
+ urls: ['example.com', 'example1.com','example2.com'] // REQUIRED.
81
+ }
82
+
83
+ ]
84
+ const mmdbPath = 'path to mmdbctl binary'
85
+ await getCrawlersIps(outputDirectory, mmdbPath, myCustomCrawlersUrls)
86
+ ```
87
+ This will compile the built in datasets with your data into a single `mmdb` database.
88
+ If `mmdbctl` is already installed in your system, simply provide `mmdbctl` for the `mmdbctl` binary argument; if not, you can run the installation wizard to install it for you automatically (see below), or you can [download](https://github.com/ipinfo/mmdbctl) it directly.
89
+
90
+
91
+
92
+ > [!TIP]
93
+ > Each data source is updated regularly, some even every 5 minutes. You can use the `-refresh` flag to restart your current datasets or use the `--refreshAll` flag to refresh your current datasets and add any missing datasets Shield-Base has to offer.
54
94
 
55
95
  ## Installation
56
96
 
@@ -81,7 +121,7 @@ shield-base
81
121
  # Or
82
122
  npx shield-base
83
123
  ```
84
- ### Programmatically
124
+ ### Programmatic
85
125
 
86
126
  ```ts
87
127
  // Core scripts
@@ -94,8 +134,9 @@ import {
94
134
  getThreatLists,
95
135
  getTorLists,
96
136
  restartData,
137
+ getCrawlersIps,
97
138
  run
98
- } from '@riavzon/shield-base"';
139
+ } from '@riavzon/shield-base';
99
140
  const contactInfo = `<name> [url] - <email>`
100
141
 
101
142
  const mmdbPath = 'path to mmdbctl binary'
@@ -110,13 +151,15 @@ const ids = [
110
151
  ]
111
152
 
112
153
  const selectedSources = true ?? ids // true for all sources, eg. lvl1, lvl2, lvl3, lvl4, Anonymous OR and array of ids
154
+
113
155
  const results = await Promise.allSettled([
114
156
  getBGPAndASN(contactInfo, outputDirectory, mmdbPath),
115
157
  buildCitiesData(outputDirectory, mmdbPath),
116
158
  getTorLists(outputDirectory, mmdbPath),
117
159
  getGeoDatas(outputDirectory, mmdbPath),
118
160
  getListOfProxies(outputDirectory, mmdbPath),
119
- getThreatLists(outputDirectory, mmdbPath, selectedSources)
161
+ getThreatLists(outputDirectory, mmdbPath, selectedSources),
162
+ getCrawlersIps(outputDirectory, mmdbPath, myCustomCrawlersUrls)
120
163
  ]);
121
164
  const restartAllData = true;
122
165
 
@@ -156,6 +199,7 @@ shield-base --l1 --l2 --acceptFireholRisk
156
199
  | `--geo` | Compile Geography data. |
157
200
  | `--proxy` | Compile Proxy data. |
158
201
  | `--tor` | Compile Tor data. |
202
+ | `--seo` | Compile verified search engine and automated agent ranges.. |
159
203
  | `--l1` | Compile FireHOL Level 1. |
160
204
  | `--l2` | Compile FireHOL Level 2. |
161
205
  | `--l3` | Compile FireHOL Level 3. |
@@ -169,7 +213,7 @@ You can read from a compiled databases via the command line with [`mmdbctl`](htt
169
213
  ```bash
170
214
  mmdbctl read -f json-pretty 8.8.8.8 outputDirectory/asn.mmdb
171
215
  ```
172
- Or with a specialized library such as [`mmdb-lib`](https://www.npmjs.com/package/mmdb-lib?activeTab=readme) or [`@maxmind/geoip2-node`](https://www.npmjs.com/package/@maxmind/geoip2-node)
216
+ Or with a specialized library such as [`mmdb-lib`](https://www.npmjs.com/package/mmdb-lib), [`@maxmind/geoip2-node`](https://www.npmjs.com/package/@maxmind/geoip2-node), or [`maxmind`](https://www.npmjs.com/package/maxmind).
173
217
 
174
218
  Example:
175
219
 
@@ -187,6 +231,20 @@ const read = await Reader.open(path.join(__dirname, '/path/to/data.mmdb'));
187
231
 
188
232
  ## Data Examples
189
233
 
234
+
235
+ ### SEO Bots
236
+ ```bash
237
+ mmdbctl read -f json-pretty 66.249.66.1 outputDirectory/goodBots.mmdb
238
+ ```
239
+ ```json
240
+ {
241
+ "provider": "google",
242
+ "range": "66.249.66.0/24",
243
+ "syncToken": "1710000000",
244
+ "creationTime": "2024-03-09T22:00:00.000Z"
245
+ }
246
+ ```
247
+
190
248
  ### BGP
191
249
  ```bash
192
250
  mmdbctl read -f json-pretty 8.8.8.8 outputDirectory/asn.mmdb
@@ -196,12 +254,12 @@ Output:
196
254
 
197
255
  ```json
198
256
  {
199
- "asn_id": "15169",
200
- "asn_name": "Google LLC",
201
- "classification": "Content",
202
- "hits": "2679",
203
- "ip": "8.8.8.8",
204
- "network": "8.8.8.0/24"
257
+ "asn_id": "15169",
258
+ "asn_name": "Google LLC",
259
+ "classification": "Content",
260
+ "hits": "2679",
261
+ "ip": "8.8.8.8",
262
+ "network": "8.8.8.0/24"
205
263
  }
206
264
  ```
207
265
 
@@ -214,31 +272,31 @@ Output:
214
272
 
215
273
  ```json
216
274
  {
217
- "capital": "Paris",
218
- "city": "Paris",
219
- "continent": "Europe",
220
- "country_code": "FR",
221
- "currency": "EUR",
222
- "currency_name": "Euro",
223
- "emoji": "🇫🇷",
224
- "ip": "137.174.48.5",
225
- "languages": "Frañs",
226
- "latitude": "48.85661400",
227
- "longitude": "2.35222190",
228
- "name": "France",
229
- "nationality": "French",
230
- "native": "France",
231
- "network": "137.174.48.0/21",
232
- "numericCode": "250",
233
- "phone": "33",
234
- "region": "FR-75C",
235
- "state": "Paris",
236
- "subregion": "Western Europe",
237
- "timeZoneName": "Central European Time",
238
- "timezone": "Europe/Paris",
239
- "tld": ".fr",
240
- "utc_offset": "UTC+01:00",
241
- "zip_code": "123456"
275
+ "capital": "Paris",
276
+ "city": "Paris",
277
+ "continent": "Europe",
278
+ "country_code": "FR",
279
+ "currency": "EUR",
280
+ "currency_name": "Euro",
281
+ "emoji": "🇫🇷",
282
+ "ip": "137.174.48.5",
283
+ "languages": "Frañs",
284
+ "latitude": "48.85661400",
285
+ "longitude": "2.35222190",
286
+ "name": "France",
287
+ "nationality": "French",
288
+ "native": "France",
289
+ "network": "137.174.48.0/21",
290
+ "numericCode": "250",
291
+ "phone": "33",
292
+ "region": "FR-75C",
293
+ "state": "Paris",
294
+ "subregion": "Western Europe",
295
+ "timeZoneName": "Central European Time",
296
+ "timezone": "Europe/Paris",
297
+ "tld": ".fr",
298
+ "utc_offset": "UTC+01:00",
299
+ "zip_code": "123456"
242
300
  }
243
301
  ```
244
302
  ### Country
@@ -250,26 +308,26 @@ Output
250
308
 
251
309
  ```json
252
310
  {
253
- "capital": "Washington",
254
- "country_code": "US",
255
- "currency": "USD",
256
- "currency_name": "United States dollar",
257
- "currency_symbol": "$",
258
- "emoji": "🇺🇸",
259
- "ip": "161.185.160.93",
260
- "languages": "Stadoù-Unanet",
261
- "name": "United States",
262
- "nationality": "American",
263
- "native": "United States",
264
- "network": "161.185.0.0-161.186.255.255",
265
- "numericCode": "840",
266
- "phone": "1",
267
- "region": "Americas",
268
- "subregion": "Northern America",
269
- "timeZoneName": "Hawaii–Aleutian Standard Time",
270
- "timezone": "America/Adak",
271
- "tld": ".us",
272
- "utc_offset": "UTC-10:00"
311
+ "capital": "Washington",
312
+ "country_code": "US",
313
+ "currency": "USD",
314
+ "currency_name": "United States dollar",
315
+ "currency_symbol": "$",
316
+ "emoji": "🇺🇸",
317
+ "ip": "161.185.160.93",
318
+ "languages": "Stadoù-Unanet",
319
+ "name": "United States",
320
+ "nationality": "American",
321
+ "native": "United States",
322
+ "network": "161.185.0.0-161.186.255.255",
323
+ "numericCode": "840",
324
+ "phone": "1",
325
+ "region": "Americas",
326
+ "subregion": "Northern America",
327
+ "timeZoneName": "Hawaii–Aleutian Standard Time",
328
+ "timezone": "America/Adak",
329
+ "tld": ".us",
330
+ "utc_offset": "UTC-10:00"
273
331
  }
274
332
  ```
275
333
  ### Tor
@@ -280,77 +338,77 @@ mmdbctl read -f json-pretty 192.42.116.52 outputDirectory/tor.mmdb
280
338
 
281
339
  ```json
282
340
  {
283
- "as": "AS215125",
284
- "as_name": "Church of Cyberology",
285
- "contact": "email:mail[]nothingtohide.nl url:nothingtohide.nl proof:uri-rsa abuse:abuse[]nothingtohide.nl ciissversion:2",
286
- "country": "nl",
287
- "country_name": "Netherlands",
288
- "exit_addresses": "192.42.116.49",
289
- "exit_policy": "reject 0.0.0.0/8:*,reject 169.254.0.0/16:*,reject 127.0.0.0/8:*,reject 192.168.0.0/16:*,reject 10.0.0.0/8:*,reject 172.16.0.0/12:*,reject 192.42.116.49:*,accept *:80,accept *:443,accept *:8080,accept *:8443,accept *:110,accept *:143,accept *:220,accept *:465,accept *:587,accept *:993,accept *:995,accept *:43,accept *:53,accept *:853,accept *:4321,accept *:11371,accept *:873,accept *:3690,accept *:9418,accept *:194,accept *:6660-6669,accept *:6679,accept *:6697,accept *:7000-7001,accept *:5222-5223,accept *:5228,accept *:64738,accept *:1194,accept *:1293,accept *:51820,accept *:8233,accept *:8333,accept *:9333,accept *:18080-18081,accept *:30303,accept *:51235,reject *:*",
290
- "exit_policy_summary": "{\"accept\":[\"43\",\"53\",\"80\",\"110\",\"143\",\"194\",\"220\",\"443\",\"465\",\"587\",\"853\",\"873\",\"993\",\"995\",\"1194\",\"1293\",\"3690\",\"4321\",\"5222-5223\",\"5228\",\"6660-6669\",\"6679\",\"6697\",\"7000-7001\",\"8080\",\"8233\",\"8333\",\"8443\",\"9333\",\"9418\",\"11371\",\"18080-18081\",\"30303\",\"51235\",\"51820\",\"64738\"]}",
291
- "exit_probability": 0.0005507535,
292
- "first_seen": "2023-07-07 00:00:00",
293
- "flags": "Exit,Fast,Running,Valid",
294
- "guard_probability": 0,
295
- "ip": "192.42.116.52",
296
- "last_changed_address_or_port": "2026-03-07 19:00:00",
297
- "last_restarted": "2026-03-07 17:44:13",
298
- "last_seen": "2026-03-08 10:00:00",
299
- "measured": true,
300
- "middle_probability": 0,
301
- "network": "192.42.116.0/24",
302
- "or_addresses": "192.42.116.49:9004",
303
- "recommended_version": true,
304
- "running": true,
305
- "version_status": "recommended"
341
+ "as": "AS215125",
342
+ "as_name": "Church of Cyberology",
343
+ "contact": "email:mail[]nothingtohide.nl url:nothingtohide.nl proof:uri-rsa abuse:abuse[]nothingtohide.nl ciissversion:2",
344
+ "country": "nl",
345
+ "country_name": "Netherlands",
346
+ "exit_addresses": "192.42.116.49",
347
+ "exit_policy": "reject 0.0.0.0/8:*,reject 169.254.0.0/16:*,reject 127.0.0.0/8:*,reject 192.168.0.0/16:*,reject 10.0.0.0/8:*,reject 172.16.0.0/12:*,reject 192.42.116.49:*,accept *:80,accept *:443,accept *:8080,accept *:8443,accept *:110,accept *:143,accept *:220,accept *:465,accept *:587,accept *:993,accept *:995,accept *:43,accept *:53,accept *:853,accept *:4321,accept *:11371,accept *:873,accept *:3690,accept *:9418,accept *:194,accept *:6660-6669,accept *:6679,accept *:6697,accept *:7000-7001,accept *:5222-5223,accept *:5228,accept *:64738,accept *:1194,accept *:1293,accept *:51820,accept *:8233,accept *:8333,accept *:9333,accept *:18080-18081,accept *:30303,accept *:51235,reject *:*",
348
+ "exit_policy_summary": "{\"accept\":[\"43\",\"53\",\"80\",\"110\",\"143\",\"194\",\"220\",\"443\",\"465\",\"587\",\"853\",\"873\",\"993\",\"995\",\"1194\",\"1293\",\"3690\",\"4321\",\"5222-5223\",\"5228\",\"6660-6669\",\"6679\",\"6697\",\"7000-7001\",\"8080\",\"8233\",\"8333\",\"8443\",\"9333\",\"9418\",\"11371\",\"18080-18081\",\"30303\",\"51235\",\"51820\",\"64738\"]}",
349
+ "exit_probability": 0.0005507535,
350
+ "first_seen": "2023-07-07 00:00:00",
351
+ "flags": "Exit,Fast,Running,Valid",
352
+ "guard_probability": 0,
353
+ "ip": "192.42.116.52",
354
+ "last_changed_address_or_port": "2026-03-07 19:00:00",
355
+ "last_restarted": "2026-03-07 17:44:13",
356
+ "last_seen": "2026-03-08 10:00:00",
357
+ "measured": true,
358
+ "middle_probability": 0,
359
+ "network": "192.42.116.0/24",
360
+ "or_addresses": "192.42.116.49:9004",
361
+ "recommended_version": true,
362
+ "running": true,
363
+ "version_status": "recommended"
306
364
  }
307
365
  {
308
- "as": "AS29802",
309
- "as_name": "HIVELOCITY, Inc.",
310
- "contact": "Unknown",
311
- "country": "us",
312
- "country_name": "United States of America",
313
- "exit_addresses": "",
314
- "exit_policy": "reject *:*",
315
- "exit_policy_summary": "{\"reject\":[\"1-65535\"]}",
316
- "exit_probability": 0,
317
- "first_seen": "2019-02-18 00:00:00",
318
- "flags": "Fast,Guard,HSDir,Running,Stable,V2Dir,Valid",
319
- "guard_probability": 0.000067750596,
320
- "ip": "66.206.0.138",
321
- "last_changed_address_or_port": "2024-01-29 19:00:00",
322
- "last_restarted": "2025-05-29 16:27:56",
323
- "last_seen": "2026-03-08 10:00:00",
324
- "measured": true,
325
- "middle_probability": 0.000039495306,
326
- "network": "66.206.0.0/24",
327
- "or_addresses": "66.206.0.82:9001",
328
- "recommended_version": false,
329
- "running": true,
330
- "version_status": "obsolete"
366
+ "as": "AS29802",
367
+ "as_name": "HIVELOCITY, Inc.",
368
+ "contact": "Unknown",
369
+ "country": "us",
370
+ "country_name": "United States of America",
371
+ "exit_addresses": "",
372
+ "exit_policy": "reject *:*",
373
+ "exit_policy_summary": "{\"reject\":[\"1-65535\"]}",
374
+ "exit_probability": 0,
375
+ "first_seen": "2019-02-18 00:00:00",
376
+ "flags": "Fast,Guard,HSDir,Running,Stable,V2Dir,Valid",
377
+ "guard_probability": 0.000067750596,
378
+ "ip": "66.206.0.138",
379
+ "last_changed_address_or_port": "2024-01-29 19:00:00",
380
+ "last_restarted": "2025-05-29 16:27:56",
381
+ "last_seen": "2026-03-08 10:00:00",
382
+ "measured": true,
383
+ "middle_probability": 0.000039495306,
384
+ "network": "66.206.0.0/24",
385
+ "or_addresses": "66.206.0.82:9001",
386
+ "recommended_version": false,
387
+ "running": true,
388
+ "version_status": "obsolete"
331
389
  }
332
390
  ```
333
391
  ### Proxy
334
392
  ```bash
335
- mmdbctl read -f json-pretty 1.0.136.198 outputDirectory/proxy.mmdb
393
+ mmdbctl read -f json-pretty 1.0.136.198 outputDirectory/proxy.mmdb
336
394
 
337
- mmdbctl read -f json-pretty 102.217.190.157 outputDirectory/proxy.mmdb
395
+ mmdbctl read -f json-pretty 102.217.190.157 outputDirectory/proxy.mmdb
338
396
 
339
397
  ```
340
398
  Output:
341
399
  ```json
342
400
  {
343
- "comment": "Ip from Firehol",
344
- "ip": "1.0.136.198",
345
- "network": "1.0.136.198/32",
346
- "port": "unknown"
401
+ "comment": "Ip from Firehol",
402
+ "ip": "1.0.136.198",
403
+ "network": "1.0.136.198/32",
404
+ "port": "unknown"
347
405
  }
348
406
 
349
407
  {
350
- "comment": "TheSpeedX, openproxy",
351
- "ip": "102.217.190.157",
352
- "network": "102.217.190.157/32",
353
- "port": "7080"
408
+ "comment": "TheSpeedX, openproxy",
409
+ "ip": "102.217.190.157",
410
+ "network": "102.217.190.157/32",
411
+ "port": "7080"
354
412
  }
355
413
  ```
356
414
  ### Threat levels
@@ -368,32 +426,32 @@ Output:
368
426
 
369
427
  ```json
370
428
  {
371
- "comment": "firehol_l1 Maintainer: http://iplists.firehol.org/",
372
- "ip": "45.143.203.111",
373
- "network": "45.143.203.0/24"
429
+ "comment": "firehol_l1 Maintainer: http://iplists.firehol.org/",
430
+ "ip": "45.143.203.111",
431
+ "network": "45.143.203.0/24"
374
432
  }
375
433
 
376
434
  {
377
- "comment": "firehol_l2 Maintainer: http://iplists.firehol.org/",
378
- "ip": "1.31.80.222",
379
- "network": "1.31.80.222/32"
435
+ "comment": "firehol_l2 Maintainer: http://iplists.firehol.org/",
436
+ "ip": "1.31.80.222",
437
+ "network": "1.31.80.222/32"
380
438
  }
381
439
  {
382
- "comment": "firehol_l3 Maintainer: http://iplists.firehol.org/",
383
- "ip": "1.24.16.177",
384
- "network": "1.24.16.177/32"
440
+ "comment": "firehol_l3 Maintainer: http://iplists.firehol.org/",
441
+ "ip": "1.24.16.177",
442
+ "network": "1.24.16.177/32"
385
443
  }
386
444
 
387
445
  {
388
- "comment": "firehol_l4 Maintainer: http://iplists.firehol.org/",
389
- "ip": "1.10.141.115",
390
- "network": "1.10.141.115/32"
446
+ "comment": "firehol_l4 Maintainer: http://iplists.firehol.org/",
447
+ "ip": "1.10.141.115",
448
+ "network": "1.10.141.115/32"
391
449
  }
392
450
 
393
451
  {
394
- "comment": "firehol_anonymous Maintainer: http://iplists.firehol.org/",
395
- "ip": "1.0.136.76",
396
- "network": "1.0.136.76/32"
452
+ "comment": "firehol_anonymous Maintainer: http://iplists.firehol.org/",
453
+ "ip": "1.0.136.76",
454
+ "network": "1.0.136.76/32"
397
455
  }
398
456
  ```
399
457
 
package/dist/chunk-MZE5OD6T.js ADDED
@@ -0,0 +1,53 @@
1
+ import {anyOf,exactly,charIn,digit,letter,createRegExp,oneOrMore,charNotIn}from'magic-regexp';import*as N from'fs';import N__default,{existsSync}from'fs';import Ht from'child_process';import qt from'util';import*as G from'path';import G__default from'path';import U,{consola}from'consola';import {fileURLToPath}from'url';import et from'ipaddr.js';import {IpRange,IpAddress}from'cidr-calc';import ve from'os';import {readFile}from'fs/promises';var Vt=qt.promisify(Ht.exec),R=async(o,s={})=>{try{let{stdout:a,stderr:e}=await Vt(o,{...s,maxBuffer:s.maxBuffer??10485760}),n={stdout:typeof a=="string"?a.trim():a,stderr:typeof e=="string"?e.trim():e};return s.silent||(console.log(`[run]: ${o}`),n.stdout&&console.log("stdout:",n.stdout),n.stderr&&console.error("stderr:",n.stderr)),n}catch(a){if(console.error(`Execution failed for command: "${o}"`),typeof a=="object"&&a!==null){let e=a;e.stderr&&console.error("Error Output:",e.stderr);}throw a}};var k=consola.withTag("[ASN/BGP]");async function rt(o,s,a){let e=["https://bgp.tools/asns.csv","https://bgp.tools/table.jsonl"],n=G__default.resolve(s,"asn.mmdb"),c=G__default.resolve(s,"temp_asn_data.json");try{k.info(`
2
+ Fetching ASN Dictionary from BGP.tools...`);let m=await(await fetch(e[0],{method:"GET",headers:{"User-Agent":o}})).text(),i=new Map,l=m.split(`
3
+ `).slice(1),y=createRegExp(exactly("AS").and(oneOrMore(digit).groupedAs("asnNumber"))),t=charNotIn('"').times.any(),d=t.and(exactly('"')).times(2),x=createRegExp(exactly(",").before(d.times.any().and(t).at.lineEnd()));for(let b of l){if(!b)continue;let _=b.split(x),E=_[0].match(y),h=E?.groups.asnNumber?parseInt(E.groups.asnNumber,10):null,$=createRegExp(exactly('"').at.lineStart().or(exactly('"').at.lineEnd()),["g"]),P=_[1]?.replace($,"").trim()||"Unknown",f=_[2]?.trim()||"Unknown";h!==null&&i.set(h,{name:P,type:f});}k.success(`SUCCESS: Loaded ${String(i.size)} ASN definitions.`),k.info("Fetching Routing Table...");let S=await(await fetch(e[1],{method:"GET",headers:{"User-Agent":o}})).text();k.info("Mapping CIDR to ASN data...");let O=S.split(`
4
+ `),u=[];for(let b of O){if(!b)continue;let _=JSON.parse(b);if(_.Hits<10)continue;let E=i.get(_.ASN)??{name:"Unknown",type:"Unknown"},h={range:_.CIDR,asn_id:String(_.ASN),asn_name:E.name,classification:E.type,hits:String(_.Hits)};u.push(JSON.stringify(h));}u.length>0&&k.log("SAMPLE:",u[0]),k.info(`Writing ${String(u.length)} enriched entries to temporary JSON...`),N.writeFileSync(c,u.join(`
5
+ `),"utf-8"),k.info(`Compiling MMDB with mmdbctl to ${n}...`);let w=`${a} import --in ${c} --out ${n}`,p=await R(w);p.stdout&&k.log(`mmdbctl: ${p.stdout.toString().trim()}`),k.success(`COMPLETED: Successfully compiled ASN MMDB to ${n}
6
+ `);}catch(r){k.error(`
7
+ ERROR during processing:`,r),process.exit(1);}finally{N.existsSync(c)&&N.unlinkSync(c);}}var gt=G.dirname(fileURLToPath(import.meta.url)),re=[G.resolve(gt,"./countries+states+cities.json"),G.resolve(gt,"../../public/countries+states+cities.json")].find(o=>N.existsSync(o))??G.resolve(gt,"./countries+states+cities.json"),B=consola.withTag("[CITY/GEO]");async function st(o,s){B.info(`
8
+ Building global geographic index from hierarchical database...`);let a=G.resolve(o,"city.mmdb"),e=G.resolve(o,"temp_city_data.json"),n=N.readFileSync(re,"utf8"),c=JSON.parse(n),r=new Map;for(let m of c){let i=new Map;if(m.states)for(let l of m.states){let y=new Map;if(l.cities)for(let t of l.cities)y.set(t.name.toLowerCase(),t);i.set(l.state_code,{meta:l,cities:y});}r.set(m.iso2.toUpperCase(),{meta:m,states:i});}try{B.info("Fetching validated geofeed CSV...");let l=(await(await fetch("https://geolocatemuch.com/geofeeds/validated-all.csv")).text()).split(`
9
+ `),y=createRegExp(exactly("#").at.lineStart());B.success(`SUCCESS: Received ${String(l.length)} IP ranges. Mapping to geographic index...`);let t=[];for(let g of l){if(!g.trim()||y.test(g))continue;let S=g.split(","),O=S[0]?.trim()||"",u=S[1]?.trim()||"",w=S[2]?.trim()||"",p=S[3]?.trim()||"",b=S[4]?.trim()||"";if(!u&&!p&&!b)continue;let _=u.toUpperCase()||"",E="",h="",$="",P="",f={},W=r.get(_);if(W){if(f=W.meta,E=W.meta.emoji||"",w){let Y=w.includes("-")?w.split("-").pop():w,T=W.states.get(Y);if(T){if(P=T.meta.name||T.meta.state_code||"",p){let dt=T.cities.get(p.toLowerCase());dt&&(h=dt.latitude||"",$=dt.longitude||"");}!h&&T.meta.latitude&&T.meta.longitude&&(h=T.meta.latitude||"",$=T.meta.longitude||"");}}if(!h&&p)for(let[,Y]of W.states){let T=Y.cities.get(p.toLowerCase());if(T){h=T.latitude||"",$=T.longitude||"",P||(P=Y.meta.name||Y.meta.state_code||"");break}}!h&&f.latitude&&f.longitude&&(h=f.latitude||"",$=f.longitude||"");}let Wt={range:O,country_code:u,region:w||(f.region??""),subregion:f.subregion??"",city:p,zip_code:b,latitude:h,longitude:$,state:P,name:f.name??"Unknown",native:f.native??"",phone:f.phonecode??"",numericCode:f.numeric_code??"",continent:f.region??"",capital:f.capital??"",currency:f.currency??"",currency_name:f.currency_name??"",languages:f.translations?Object.values(f.translations)[0]??"":"",emoji:E,timezone:f.timezones?.[0]?.zoneName??"",utc_offset:f.timezones?.[0]?.gmtOffsetName??"",timeZoneName:f.timezones?.[0]?.tzName??"",tld:f.tld??"",nationality:f.nationality??""};t.push(JSON.stringify(Wt));}B.info(`Writing ${String(t.length)} enriched entries to temporary JSON...`),N.writeFileSync(e,t.join(`
10
+ `),"utf-8"),B.start("Compiling MMDB with mmdbctl...");let d=`${s} import --in ${e} --out ${a}`,x=await R(d);x.stdout&&B.log(`mmdbctl: ${x.stdout.toString().trim()}`),B.success(`COMPLETED: Successfully compiled City MMDB to ${a}
11
+ `);}catch(m){B.error(`
12
+ ERROR during processing:`,m),process.exit(1);}finally{N.existsSync(e)&&N.unlinkSync(e);}}var A=U.withTag("[TOR]");async function nt(o,s){let a="https://onionoo.torproject.org/details",e=G__default.resolve(o,"tor.mmdb"),n=G__default.resolve(o,"temp_tor_nodes.json");try{A.info(`
13
+ Fetching node data from Onionoo API...`);let c=await fetch(a,{method:"GET",headers:{"Accept-Encoding":"gzip",Accept:"application/json"}});if(!c.ok){A.error(`ERROR: API fetch failed with status ${String(c.status)}`);return}let r=await c.json();A.success("SUCCESS: Data received, mapping relay nodes...");let i=(r.relays||[]).filter(t=>{let d=t.flags?.map(x=>x.toLowerCase())??[];return d.includes("exit")||d.includes("valid")||d.includes("running")||d.includes("stable")||d.includes("hsdir")&&t.exit_addresses&&t.or_addresses});if(i.length===0){A.warn("WARNING: No relay nodes found matching criteria.");return}let l=[];for(let t of i){let d=[...t.exit_addresses??[],...t.or_addresses??[]];if(d)for(let x of d){let g=x;g.startsWith("[")?g=g.split("]:")[0].substring(1):g=g.split(":")[0];let S=et.parse(g),O;if(S.kind()==="ipv4"){let u=S;O=`${et.IPv4.networkAddressFromCIDR(`${u.toString()}/24`).toString()}/24`;}else if(S.kind()==="ipv6"){let u=S;O=`${et.IPv6.networkAddressFromCIDR(`${u.toString()}/64`).toString()}/64`;}else continue;l.push({range:O,or_addresses:t.or_addresses?.join(",")??"",exit_addresses:t.exit_addresses?.join(",")??"",last_seen:t.last_seen??"",last_changed_address_or_port:t.last_changed_address_or_port??"",first_seen:t.first_seen??"",running:t.running??!1,flags:t.flags?.join(",")??"",country:t.country??"",country_name:t.country_name??"",as:t.as??"",as_name:t.as_name??"",last_restarted:t.last_restarted??"",exit_policy:t.exit_policy?.join(",")??"",exit_policy_summary:JSON.stringify(t.exit_policy_summary)||"",exit_policy_v6_summary:t.exit_policy_v6_summary?JSON.stringify(t.exit_policy_v6_summary):void 0,contact:t.contact??"Unknown",version_status:t.version_status??"",guard_probability:t.guard_probability??0,middle_probability:t.middle_probability??0,exit_probability:t.exit_probability??0,recommended_version:t.recommended_version??!1,measured:t.measured??!1});}}A.info("SAMPLE:",l[0]);let y=l.map(t=>JSON.stringify(t)).join(`
14
+ `);N.writeFileSync(n,y,"utf-8");try{A.info("Compiling MMDB with mmdbctl...");let t=await R(`${s} import -j -i ${n} -o ${e}`);t.stdout&&A.info(`mmdbctl: ${t.stdout.toString().trim()}`),A.success(`COMPLETED: Successfully compiled Tor MMDB to ${e}
15
+ `);}catch(t){A.error("ERROR: MMDB compilation failed:",t);return}}catch(c){A.error("ERROR: Unexpected failure while fetching node list:",c);return}finally{N.existsSync(n)&&N.unlinkSync(n);}}var yt=G.dirname(fileURLToPath(import.meta.url)),ae=[G.resolve(yt,"./countries+states+cities.json"),G.resolve(yt,"../../public/countries+states+cities.json")].find(o=>N.existsSync(o))??G.resolve(yt,"./countries+states+cities.json"),F=U.withTag("[GEO/COUNTRY]");async function it(o,s){F.info(`
16
+ Building country index from local database...`);let a=G.resolve(o,"country.mmdb"),e=G.resolve(o,"temp_country_data.json"),n=N.readFileSync(ae,"utf8"),c=JSON.parse(n),r=new Map;for(let i of c)r.set(i.iso2.toUpperCase(),i);F.info("Fetching global IPv4 Country mapping from Sapics...");let m="https://raw.githubusercontent.com/sapics/ip-location-db/refs/heads/main/geo-asn-country/geo-asn-country-ipv4.csv";try{let i=await fetch(m);if(!i.ok)throw new Error(`ERROR: Failed to fetch country data: ${i.statusText}`);let y=(await i.text()).split(`
17
+ `);F.success(`SUCCESS: Received ${String(y.length)} global IP ranges. Enriching data...`);let t=[];for(let g of y){if(!g.trim())continue;let S=g.split(","),O=S[0]?.trim(),u=S[1]?.trim(),w=S[2]?.trim().toUpperCase();if(!O||!u||!w)continue;let p=r.get(w),b={range:`${O}-${u}`,country_code:w,region:p?.region??"",numericCode:p?.numeric_code??"",name:p?.name??"",native:p?.native??"",phone:p?.phonecode??"",capital:p?.capital??"",currency:p?.currency??"",currency_name:p?.currency_name??"",currency_symbol:p?.currency_symbol??"",languages:p?.translations?Object.values(p.translations)[0]??"":"",emoji:p?.emoji??"",timezone:p?.timezones?.[0]?.zoneName??"",utc_offset:p?.timezones?.[0]?.gmtOffsetName??"",timeZoneName:p?.timezones?.[0]?.tzName??"",tld:p?.tld??"",nationality:p?.nationality??"",subregion:p?.subregion??""};t.push(JSON.stringify(b));}F.info(`Writing ${String(t.length)} enriched entries to temporary JSON...`),N.writeFileSync(e,t.join(`
18
+ `),"utf-8"),F.start("Compiling MMDB with mmdbctl...");let d=`${s} import --in ${e} --out ${a}`,x=await R(d);x.stdout&&F.log(`mmdbctl: ${x.stdout.toString().trim()}`),F.success(`COMPLETED: Successfully compiled Country MMDB to ${a}
19
+ `);}catch(i){F.error(`
20
+ ERROR during processing:`,i),process.exit(1);}finally{N.existsSync(e)&&N.unlinkSync(e);}}var D=U.withTag("[PROXY]");async function ct(o,s){D.log(`
21
+ Fetching initial Proxy list from Awesome-lists...`);let a=G__default.resolve(o,"proxy.mmdb"),e=G__default.resolve(o,"temp_proxy_data.json"),n=new Set,c=[],r=["https://raw.githubusercontent.com/mthcht/awesome-lists/refs/heads/main/Lists/PROXY/ALL_PROXY_Lists.csv","https://raw.githubusercontent.com/firehol/blocklist-ipsets/refs/heads/master/firehol_proxies.netset"],m=createRegExp(exactly(oneOrMore(digit),".",oneOrMore(digit),".",oneOrMore(digit),".",oneOrMore(digit)).and(exactly("/",oneOrMore(digit)).optionally()).at.lineStart().at.lineEnd());try{let i=await fetch(r[0]);if(!i.ok)throw new Error(`ERROR: Failed to fetch awesome list: ${i.statusText}`);let y=(await i.text()).split(`
22
+ `).slice(1),t=charNotIn('"').times.any(),d=t.and(exactly('"')).times(2),x=createRegExp(exactly(",").before(d.times.any().and(t).at.lineEnd())),g=createRegExp(anyOf(exactly("PROXY_ALL_"),exactly("_list.csv"),exactly('"')),["g","i"]);for(let E of y){if(!E.trim())continue;let[h,$,P]=E.split(x);if(!h||!m.test(h)||n.has(h))continue;n.add(h);let f=P.replace(g,"").trim(),W={range:`${h}/32`,port:$,comment:f};c.push(JSON.stringify(W));}D.success(`SUCCESS: Finished Awesome-list. Discovered ${String(c.length)} unique IPs.`),D.log("Fetching secondary data from FireHOL proxy list...");let S=await fetch(r[1]);if(!S.ok)throw new Error(`ERROR: Failed to fetch FireHOL list: ${S.statusText}`);let u=(await S.text()).split(`
23
+ `),w=createRegExp(exactly("#").at.lineStart()),p=0;for(let E of u){let h=E.trim();if(!h||w.test(h))continue;let $=h;if(!m.test($)||n.has($))continue;n.add($);let P={range:$,port:"unknown",comment:"Ip from Firehol"};c.push(JSON.stringify(P)),p++;}D.success(`SUCCESS: Finished FireHOL. Added ${String(p)} NEW unique proxies. Total unique: ${String(c.length)}`),c.length>0&&D.log("SAMPLE:",c[0]),D.log(`Writing ${String(c.length)} unique proxies to temporary JSON...`),N__default.writeFileSync(e,c.join(`
24
+ `),"utf-8"),D.log("Compiling MMDB with mmdbctl...");let b=`${s} import --in ${e} --out ${a}`,_=await R(b);_.stdout&&D.log(`mmdbctl: ${_.stdout.toString().trim()}`),D.success(`COMPLETED: Successfully compiled Proxy MMDB to ${a}
25
+ `);}catch(i){D.error(`
26
+ ERROR during processing:`,i),process.exit(1);}finally{N__default.existsSync(e)&&N__default.unlinkSync(e);}}var Et={firehol_anonymous:"https://github.com/firehol/blocklist-ipsets/raw/refs/heads/master/firehol_anonymous.netset",firehol_l1:"https://github.com/firehol/blocklist-ipsets/raw/refs/heads/master/firehol_level1.netset",firehol_l2:"https://github.com/firehol/blocklist-ipsets/raw/refs/heads/master/firehol_level2.netset",firehol_l3:"https://github.com/firehol/blocklist-ipsets/raw/refs/heads/master/firehol_level3.netset",firehol_l4:"https://github.com/firehol/blocklist-ipsets/raw/refs/heads/master/firehol_level4.netset"},C=U.withTag("[THREATS]");async function lt(o,s,a){let e=" Maintainer: http://iplists.firehol.org/",n=createRegExp(exactly(oneOrMore(digit),".",oneOrMore(digit),".",oneOrMore(digit),".",oneOrMore(digit)).and(exactly("/",oneOrMore(digit)).optionally()).at.lineStart().at.lineEnd()),c=createRegExp(exactly("#").at.lineStart()),r=[];if(Array.isArray(a)?r=Object.entries(Et).filter(([m])=>a.includes(m)).map(([m,i])=>({id:m,url:i})):a&&(r=Object.entries(Et).map(([m,i])=>({id:m,url:i}))),r.length===0){C.info("No valid FireHOL data sources selected. Skipping.");return}try{C.start(`Initializing fetch for ${String(r.length)} FireHOL stream(s)...`);let m=r.map(async y=>{let t=await fetch(y.url);return {id:y.id,res:t}}),i=await Promise.allSettled(m);C.success(`SUCCESS: Fetched ${String(i.length)} data streams from FireHOL.`);let l=0;for(let y of i){if(y.status==="rejected"){C.error("Promise rejected:",y.reason);continue}let{id:t,res:d}=y.value,x=`${t} ${e}`,g=[];if(C.info(`Processing stream: ${t}...`),!d.ok){C.error(`Failed fetching ${t}: ${d.statusText}`);continue}let O=(await d.text()).split(`
27
+ `);for(let u of O){let w=u.trim();if(!w||c.test(w))continue;let p=w;if(!n.test(p))continue;let b={range:p,comment:x};l++,g.push(JSON.stringify(b));}if(C.info(`${t} summary:
28
+
29
+ \u2192 IPs Discovered: ${String(g.length)}
30
+
31
+ \u2192 Rolling Total: ${String(l)}
32
+ `),g.length>0){C.info(`SAMPLE: ${g[0]}`),C.info(`Writing entries to temporary JSON: temp_${t}.json`);let u=G__default.resolve(o,`temp_${t}.json`),w=G__default.resolve(o,`${t}.mmdb`);N__default.writeFileSync(u,g.join(`
33
+ `),"utf-8"),console.log(`Compiling MMDB with mmdbctl to ${w}...`);let p=`${s} import --in ${u} --out ${w}`,b=await R(p);b.stdout&&C.box(`mmdbctl: ${b.stdout.toString().trim()}`),N__default.existsSync(u)&&N__default.unlinkSync(u),C.success(`SUCCESS: Compiled ${t}.mmdb
34
+ `);}else C.warn(`WARNING: No valid IPs found for ${t}. Skipping.
35
+ `);}C.success(`COMPLETED: All threat intelligence sources processed successfully.
36
+ `);}catch(m){C.error(`
37
+ FATAL ERROR during processing:`,m),process.exit(1);}}var Tt=[{name:"google",type:"JSON",urls:["https://developers.google.com/static/search/apis/ipranges/googlebot.json","https://developers.google.com/static/search/apis/ipranges/special-crawlers.json","https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json","https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json","https://www.gstatic.com/ipranges/goog.json"]},{name:"bing",type:"JSON",urls:["https://www.bing.com/toolbox/bingbot.json"]},{name:"openai",type:"JSON",urls:["https://openai.com/gptbot.json","https://openai.com/chatgpt-user.json","https://openai.com/searchbot.json"]},{name:"apple",type:"JSON",urls:["https://search.developer.apple.com/applebot.json"]},{name:"ahrefs",type:"JSON",urls:["https://api.ahrefs.com/v3/public/crawler-ip-ranges"]},{name:"duckDuckGo",type:"HTML",urls:["https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot","https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot"]},{name:"commonCrawler",type:"HTML",urls:["https://commoncrawl.org/faq"]},{name:"xAndTwitter",type:"HTML",urls:["https://developer.x.com/en/docs/x-for-websites/cards/guides/troubleshooting-cards"]},{name:"facebook",type:"CSV",urls:["https://www.facebook.com/peering/geofeed"]},{name:"pinterest",type:"HTML",urls:["https://help.pinterest.com/en/business/article/pinterestbot"]},{name:"telegram",type:"HTML",urls:["https://core.telegram.org/bots/webhooks"]},{name:"semrush",type:"HTML",urls:["https://www.semrush.com/kb/1149-issues-with-crawling-a-domain"]}];var ot=o=>{if(!o)throw new Error("Empty IP string");if(o.includes("/")){let[s,a]=o.split("/");if(!et.isValidCIDR(o))throw new Error("Invalid CIDR");return `${et.parse(s).toString()}/${a}`}if(!et.isValid(o))throw new Error("Invalid Ip address");return et.parse(o).toString()},kt=(o,s,a)=>{let e=ot(o);if(s&&!a){let n=ot(s);return new IpRange(IpAddress.of(e),IpAddress.of(n)).toCidrs().map(r=>r.toString())}if(a){let n=`${e}/${a}`;if(!et.isValidCIDR(n))throw new Error("Invalid Extracted CIDR");return n}return e};var ue=U.withTag("GOOD-BOT-CRAWLER");async function At(o,s){ue.start(`Processing provider: ${o}...`);let a=s.map(async n=>{let c=await fetch(n);if(!c.ok)throw new Error(`fetch ${o} ${n} -> ${String(c.status)}`);let r=await c.json();return Array.isArray(r)?r:[r]});return (await Promise.all(a)).flat()}var bt=U.withTag("GOOD-BOT-CRAWLER"),Dt=false;async function he(o){if(!Dt)try{await R("command -v curl",{silent:!0}),Dt=!0;}catch{throw bt.error("curl needed to be installed to run goodBots scrapper in order to avoid being blocked."),new Error("Fatal: curl is required for fallback scraping.")}try{let{stdout:s}=await R(`curl -sL ${o}`,{silent:!0});return typeof s=="string"?s:s.toString("utf-8")}catch(s){return bt.warn(`curl failed for ${o}: ${String(s)}`),""}}async function Mt(o){try{let s=await fetch(o,{headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",Accept:"text/html,text/csv,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}});if(!s.ok)throw new Error(`Failed to fetch ${o}: ${s.statusText} - ${String(s.status)}`);return await s.text()}catch(s){return bt.warn(`Fetch failed for ${o}:
38
+ ${String(s)},
39
+ Using curl instead...`),await he(o)}}var mt=anyOf(exactly("25",charIn("012345")),exactly("2",charIn("01234"),digit),exactly(charIn("01"),digit,digit),exactly(digit,digit),digit),It=anyOf(exactly(mt,".",mt,".",mt,".",mt)),M=anyOf(digit,charIn("abcdefABCDEF")).times.between(1,4),Se=anyOf(exactly(M,":").times(7).and(M),exactly(exactly(M,":").times.between(3,7),":"),exactly(M,":").times.between(1,5).and("::").and(exactly(M,":").times.any()).and(M),exactly("::").and(anyOf(M,It,exactly(M,":").times.any().and(M))),exactly(M,":").times.between(1,7).and(":")),Nt=anyOf(It,Se).notAfter(anyOf(letter,digit,":")),jt=createRegExp(Nt.groupedAs("ip1"),anyOf(exactly("/",digit.times.between(1,3).groupedAs("mask")),exactly(anyOf(" to "," - ","-"," and "),Nt.groupedAs("ip2"))).optionally(),["g","i"]);var Lt=U.withTag("GOOD-BOT-CRAWLER");async function Bt(o,s){Lt.start(`Processing provider: ${s}...`);let e=(await Promise.all(o.map(Mt))).flatMap(n=>Array.from(n.matchAll(jt)).flatMap(r=>{if(!r.groups?.ip1)return [];try{let m=kt(r.groups.ip1,r.groups.ip2,r.groups.mask);return Array.isArray(m)?m:[m]}catch(m){return Lt.warn(`Failed to normalize extracted IP match from ${s}:`,r[0],`Message: ${String(m)}`),[]}}));return [{creationTime:new Date().toISOString(),prefixes:Array.from(new Set(e)).map(n=>({ipv4Prefix:n}))}]}var H=U.withTag("GOOD-BOT-CRAWLER");async function Jt(o,s,a){H.start("Starting Good Bots IP scraper...");let e=G__default.resolve(o,"./goodBots.mmdb"),n=[...Tt,...a??[]];a&&a.length>0&&a.forEach(l=>{if(!l.name||!l.type||!l.urls)throw new Error(`Provider ${l.name||"unknown"} is missing required fields.`)});let c={};for(let l of n)l.type==="JSON"?c[l.name]=await At(l.name,l.urls):c[l.name]=await Bt(l.urls,l.name);U.info("Building data...");let r=Object.entries(c).flatMap(([l,y])=>y.flatMap(t=>{if(typeof t=="string")try{return [{range:ot(t),provider:l,syncToken:"unknown",creationTime:new Date().toISOString()}]}catch(d){return H.warn(`Invalid ip format skipping: ${t} -> ${String(d)}`),null}return t.prefixes.map(d=>{let x=d.ipv4Prefix??d.ipv6Prefix??"";if(!x)return null;try{return {range:ot(x),provider:l,syncToken:t.syncToken,creationTime:t.creationTime}}catch(g){return H.warn(`Invalid ip format skipping from prefix: ${x} -> ${String(g)}`),null}}).filter(Boolean)}));H.info(`Total records to import: ${String(r.length)}`);let m=G__default.resolve(o,"temp_bots.json"),i=r.map(l=>JSON.stringify(l)).join(`
40
+ `);N__default.writeFileSync(m,i,"utf-8");try{H.info("Compiling MMDB...");let l=`${s} import --in ${m} --out ${e}`;await R(l),H.success(`Successfully generated ${e}`);}catch(l){throw H.error(`Failed to compile MMDB: ${String(l)}`),l}finally{N__default.existsSync(m)&&N__default.unlinkSync(m);}}var ft=U.withTag("Shield Base");async function zt(o,s,a,e){ft.box(`
41
+ =========================================
42
+ === Starting data generation pipeline ===
43
+ =========================================
44
+ `);try{(await Promise.allSettled([rt(s,o,e),st(o,e),nt(o,e),it(o,e),ct(o,e),lt(o,e,a),Jt(o,e)])).forEach((c,r)=>{c.status==="rejected"&&ft.error(`Script cluster [${String(r)}] failed:`,String(c.reason));}),ft.info(`
45
+ ====================================
46
+ Pipeline execution finished.
47
+ ====================================
48
+ `);}catch(n){ft.error(`
49
+ ====================================
50
+ Fatal error in pipeline: ${String(n)}
51
+ ====================================
52
+ `);}}var Ut=G__default.join(ve.homedir(),".shield-base",".cache.json");async function qo(o,s){existsSync(Ut)||(U.error("No cache found. Cannot restart. Please run a standard compilation first."),process.exit(1));let a=await readFile(Ut,"utf-8"),e=JSON.parse(a);if(!e.selectedDataTypes||e.selectedDataTypes.length===0){U.warn("Cache is empty. No data sources to restart.");return}if(s){U.info("Restarting ALL data sources..."),await zt(o,e.useragent,true,e.mmdbctlPath),U.success("\u2728 All data successfully refreshed!");return}U.info(`Restarting data compilation from cache: ${e.selectedDataTypes.join(", ")}`);let n=e.selectedDataTypes.filter(i=>i.startsWith("firehol_")||i==="anonymous"),c=e.selectedDataTypes.filter(i=>!i.startsWith("firehol_")&&i!=="anonymous"),r=[];c.includes("BGP")&&r.push({name:"BGP & ASN",task:()=>rt(e.useragent||"",o,e.mmdbctlPath)}),c.includes("City")&&r.push({name:"City",task:()=>st(e.outPutPath,e.mmdbctlPath)}),c.includes("Geography")&&r.push({name:"Geography",task:()=>it(e.outPutPath,e.mmdbctlPath)}),c.includes("Proxy")&&r.push({name:"Proxy",task:()=>ct(e.outPutPath,e.mmdbctlPath)}),c.includes("Tor")&&r.push({name:"Tor",task:()=>nt(e.outPutPath,e.mmdbctlPath)}),n.length>0&&r.push({name:"Threats",task:()=>lt(e.outPutPath,e.mmdbctlPath,n)}),U.start(`Running ${String(r.length)} restart jobs...`),(await Promise.allSettled(r.map(i=>i.task()))).forEach((i,l)=>{i.status==="rejected"&&U.error(`[${r[l].name}] Failed:`,i.reason);}),U.success("\u2728 Cached data successfully refreshed!");}export{R as a,rt as b,st as c,nt as d,it as e,ct as f,lt as g,ot as h,kt as i,Nt as j,jt as k,Jt as l,zt as m,qo as n};//# sourceMappingURL=chunk-MZE5OD6T.js.map
53
+ //# sourceMappingURL=chunk-MZE5OD6T.js.map