cloudflare-bot-directory 0.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -25
- package/package.json +34 -11
- package/src/index.d.ts +58 -0
- package/src/index.json +20 -60
- package/.editorconfig +0 -19
- package/.gitattributes +0 -1
- package/.github/dependabot.yml +0 -11
- package/.github/workflows/cron.yml +0 -32
- package/.github/workflows/main.yml +0 -68
- package/.github/workflows/pull_request.yml +0 -36
- package/scripts/fetch-bots.mjs +0 -29
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# CloudFlare Bot Directory
|
|
2
2
|
|
|
3
3
|
<p align="center">
|
|
4
4
|
<br>
|
|
@@ -6,43 +6,70 @@
|
|
|
6
6
|
<br>
|
|
7
7
|
</p>
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
|
|
10
|
+
[](https://www.npmjs.org/package/cloudflare-bot-directory)
|
|
11
|
+
|
|
12
|
+
> A comprehensive list of **500+ verified bots** and web crawlers from [CloudFlare Radar](https://radar.cloudflare.com/traffic/verified-bots), available as a JSON dataset for bot detection, user agent analysis, and web scraping identification.
|
|
13
|
+
|
|
14
|
+
## Why
|
|
15
|
+
|
|
16
|
+
Identifying legitimate bots from malicious scrapers is essential for web security and analytics. This package provides the official CloudFlare Radar verified bots directory, helping you:
|
|
17
|
+
|
|
18
|
+
- **Detect verified bots** – Identify legitimate crawlers like Googlebot, Bingbot, and more
|
|
19
|
+
- **Filter analytics** – Exclude known bots from your traffic reports
|
|
20
|
+
- **Allow-list crawlers** – Permit verified bots while blocking suspicious traffic
|
|
21
|
+
- **User agent lookup** – Match user agent strings against known bot patterns
|
|
22
|
+
|
|
23
|
+
## Data Structure
|
|
24
|
+
|
|
25
|
+
Each bot entry includes:
|
|
26
|
+
|
|
27
|
+
| Field | Description |
|
|
28
|
+
| ------------------- | ---------------------------------------------- |
|
|
29
|
+
| `slug` | URL-friendly unique identifier |
|
|
30
|
+
| `name` | Human-readable bot name |
|
|
31
|
+
| `kind` | Bot kind classification |
|
|
32
|
+
| `operator` | Company or organization operating the bot |
|
|
33
|
+
| `operatorUrl` | URL to operator's documentation |
|
|
34
|
+
| `category` | Bot category (Search Engine, Monitoring, etc.) |
|
|
35
|
+
| `description` | What the bot does |
|
|
36
|
+
| `followsRobotsTxt` | Whether the bot respects robots.txt |
|
|
37
|
+
| `userAgentPatterns` | User agent pattern(s) for matching |
|
|
38
|
+
| `userAgents` | Known user agent string(s) |
|
|
10
39
|
|
|
11
40
|
## Install
|
|
12
41
|
|
|
13
42
|
```bash
|
|
14
|
-
|
|
43
|
+
npm install cloudflare-bot-directory
|
|
15
44
|
```
|
|
16
45
|
|
|
17
46
|
## Usage
|
|
18
47
|
|
|
19
48
|
```js
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
console.log(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
//
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
// 'addsearchbot',
|
|
36
|
-
// 'addthis',
|
|
37
|
-
// 'adidxbot',
|
|
38
|
-
// 'adstxtcrawler',
|
|
39
|
-
// ... 502 more items
|
|
40
|
-
// ]
|
|
49
|
+
const bots = require('cloudflare-bot-directory')
|
|
50
|
+
|
|
51
|
+
// Get all bot slugs
|
|
52
|
+
console.log(bots.map(bot => bot.slug))
|
|
53
|
+
// ['2checkout', '360monitoring', 'googlebot', 'bingbot', ...]
|
|
54
|
+
|
|
55
|
+
// Find a specific bot
|
|
56
|
+
const googlebot = bots.find(bot => bot.slug === 'googlebot')
|
|
57
|
+
|
|
58
|
+
// Filter by category
|
|
59
|
+
const searchBots = bots.filter(bot => bot.category === 'Search Engine Crawler')
|
|
60
|
+
|
|
61
|
+
// Check if a user agent is a known bot
|
|
62
|
+
const isKnownBot = (userAgent) =>
|
|
63
|
+
bots.some(bot => userAgent.includes(bot.name))
|
|
41
64
|
```
|
|
42
65
|
|
|
66
|
+
## Related
|
|
67
|
+
|
|
68
|
+
- [top-user-agents](https://github.com/microlinkhq/top-user-agents) – An always up-to-date list of the top 100 HTTP user-agents most used over the Internet.
|
|
69
|
+
|
|
43
70
|
## License
|
|
44
71
|
|
|
45
72
|
**cloudflare-bot-directory** © [Kiko Beats](https://kikobeats.com), released under the [MIT](https://github.com/kikobeats/cloudflare-bot-directory/blob/master/LICENSE.md) License.<br>
|
|
46
73
|
Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/kikobeats/cloudflare-bot-directory/contributors).
|
|
47
74
|
|
|
48
|
-
> [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/kikobeats) ·
|
|
75
|
+
> [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/kikobeats) · X [@kikobeats](https://x.com/kikobeats)
|
package/package.json
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cloudflare-bot-directory",
|
|
3
|
-
"description": "
|
|
3
|
+
"description": "CloudFlare Radar verified bots directory - 500+ web crawlers, search engine bots, and user agents as JSON for bot detection and traffic filtering",
|
|
4
4
|
"homepage": "https://github.com/kikobeats/cloudflare-bot-directory",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "1.0.1",
|
|
6
|
+
"types": "./src/index.d.ts",
|
|
7
|
+
"main": "./src/index.js",
|
|
6
8
|
"exports": {
|
|
7
9
|
".": "./src/index.js"
|
|
8
10
|
},
|
|
@@ -11,6 +13,7 @@
|
|
|
11
13
|
"name": "Kiko Beats",
|
|
12
14
|
"url": "https://kikobeats.com"
|
|
13
15
|
},
|
|
16
|
+
"contributors": [],
|
|
14
17
|
"repository": {
|
|
15
18
|
"type": "git",
|
|
16
19
|
"url": "git+https://github.com/kikobeats/cloudflare-bot-directory.git"
|
|
@@ -19,10 +22,29 @@
|
|
|
19
22
|
"url": "https://github.com/kikobeats/cloudflare-bot-directory/issues"
|
|
20
23
|
},
|
|
21
24
|
"keywords": [
|
|
25
|
+
"bingbot",
|
|
26
|
+
"bot-detection",
|
|
27
|
+
"bot-filtering",
|
|
28
|
+
"bot-list",
|
|
22
29
|
"bots",
|
|
23
30
|
"cloudflare",
|
|
31
|
+
"cloudflare-radar",
|
|
32
|
+
"crawler",
|
|
33
|
+
"crawlers",
|
|
24
34
|
"directory",
|
|
25
|
-
"
|
|
35
|
+
"googlebot",
|
|
36
|
+
"radar",
|
|
37
|
+
"scraper",
|
|
38
|
+
"scraper-detection",
|
|
39
|
+
"search-engine",
|
|
40
|
+
"security",
|
|
41
|
+
"spam-detection",
|
|
42
|
+
"traffic-filtering",
|
|
43
|
+
"user-agent",
|
|
44
|
+
"user-agents",
|
|
45
|
+
"verified-bots",
|
|
46
|
+
"web-crawler",
|
|
47
|
+
"web-security"
|
|
26
48
|
],
|
|
27
49
|
"devDependencies": {
|
|
28
50
|
"@commitlint/cli": "latest",
|
|
@@ -36,6 +58,7 @@
|
|
|
36
58
|
"git-authors-cli": "latest",
|
|
37
59
|
"github-generate-release": "latest",
|
|
38
60
|
"nano-staged": "latest",
|
|
61
|
+
"p-map": "latest",
|
|
39
62
|
"simple-git-hooks": "latest",
|
|
40
63
|
"standard": "latest",
|
|
41
64
|
"standard-version": "latest"
|
|
@@ -43,6 +66,9 @@
|
|
|
43
66
|
"engines": {
|
|
44
67
|
"node": ">= 20"
|
|
45
68
|
},
|
|
69
|
+
"files": [
|
|
70
|
+
"src"
|
|
71
|
+
],
|
|
46
72
|
"license": "MIT",
|
|
47
73
|
"commitlint": {
|
|
48
74
|
"extends": [
|
|
@@ -67,23 +93,20 @@
|
|
|
67
93
|
"commit-msg": "npx commitlint --edit",
|
|
68
94
|
"pre-commit": "npx nano-staged"
|
|
69
95
|
},
|
|
70
|
-
"dependencies": {
|
|
71
|
-
"p-map": "~7.0.4"
|
|
72
|
-
},
|
|
73
96
|
"scripts": {
|
|
74
97
|
"clean": "rm -rf node_modules",
|
|
75
98
|
"contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
|
|
76
99
|
"coverage": "c8 report --reporter=text-lcov > coverage/lcov.info",
|
|
77
|
-
"lint": "standard
|
|
100
|
+
"lint": "standard",
|
|
78
101
|
"postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
|
|
79
102
|
"pretest": "npm run lint",
|
|
80
103
|
"release": "pnpm run release:version && pnpm run release:changelog && pnpm run release:commit && pnpm run release:tag",
|
|
81
|
-
"release:github": "github-generate-release",
|
|
82
|
-
"release:tags": "git push origin HEAD:master --follow-tags",
|
|
83
|
-
"test": "c8 ava",
|
|
84
104
|
"release:changelog": "conventional-changelog -p conventionalcommits -i CHANGELOG.md -s",
|
|
85
105
|
"release:commit": "git add package.json CHANGELOG.md && git commit -m \"chore(release): $(node -p \"require('./package.json').version\")\"",
|
|
106
|
+
"release:github": "github-generate-release",
|
|
86
107
|
"release:tag": "git tag -a v$(node -p \"require('./package.json').version\") -m \"v$(node -p \"require('./package.json').version\")\"",
|
|
87
|
-
"release:
|
|
108
|
+
"release:tags": "git push origin HEAD:master --follow-tags",
|
|
109
|
+
"release:version": "standard-version --skip.changelog --skip.commit --skip.tag",
|
|
110
|
+
"test": "c8 ava"
|
|
88
111
|
}
|
|
89
112
|
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bot category types from CloudFlare Radar
|
|
3
|
+
*/
|
|
4
|
+
export type BotCategory =
|
|
5
|
+
| 'ACADEMIC_RESEARCH'
|
|
6
|
+
| 'ACCESSIBILITY'
|
|
7
|
+
| 'ADVERTISING_AND_MARKETING'
|
|
8
|
+
| 'AGGREGATOR'
|
|
9
|
+
| 'AI_ASSISTANT'
|
|
10
|
+
| 'AI_CRAWLER'
|
|
11
|
+
| 'AI_SEARCH'
|
|
12
|
+
| 'ARCHIVER'
|
|
13
|
+
| 'FEED_FETCHER'
|
|
14
|
+
| 'MONITORING_AND_ANALYTICS'
|
|
15
|
+
| 'OTHER'
|
|
16
|
+
| 'PAGE_PREVIEW'
|
|
17
|
+
| 'SEARCH_ENGINE_CRAWLER'
|
|
18
|
+
| 'SEARCH_ENGINE_OPTIMIZATION'
|
|
19
|
+
| 'SECURITY'
|
|
20
|
+
| 'SOCIAL_MEDIA_MARKETING'
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Bot kind types
|
|
24
|
+
*/
|
|
25
|
+
export type BotKind = 'BOT' | 'AGENT'
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A verified bot entry from CloudFlare Radar Bot Directory
|
|
29
|
+
*/
|
|
30
|
+
export interface Bot {
|
|
31
|
+
/** URL-friendly unique identifier */
|
|
32
|
+
slug: string
|
|
33
|
+
/** Human-readable bot name */
|
|
34
|
+
name: string
|
|
35
|
+
/** Bot kind classification */
|
|
36
|
+
kind: BotKind
|
|
37
|
+
/** Company or organization operating the bot */
|
|
38
|
+
operator: string | null
|
|
39
|
+
/** URL to operator's documentation or website */
|
|
40
|
+
operatorUrl: string | null
|
|
41
|
+
/** Bot category classification */
|
|
42
|
+
category: BotCategory
|
|
43
|
+
/** Description of what the bot does */
|
|
44
|
+
description: string | null
|
|
45
|
+
/** Whether the bot respects robots.txt */
|
|
46
|
+
followsRobotsTxt: boolean
|
|
47
|
+
/** User agent pattern(s) for matching */
|
|
48
|
+
userAgentPatterns: string[]
|
|
49
|
+
/** Known user agent string(s) */
|
|
50
|
+
userAgents: string[]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Array of all verified bots from CloudFlare Radar Bot Directory
|
|
55
|
+
*/
|
|
56
|
+
declare const bots: Bot[]
|
|
57
|
+
|
|
58
|
+
export default bots
|
package/src/index.json
CHANGED
|
@@ -252,9 +252,7 @@
|
|
|
252
252
|
"category": "AI_ASSISTANT",
|
|
253
253
|
"description": "The AGI Agent is a productivity assistant that takes actions and makes purchases on behalf of users",
|
|
254
254
|
"followsRobotsTxt": false,
|
|
255
|
-
"userAgentPatterns": [
|
|
256
|
-
""
|
|
257
|
-
],
|
|
255
|
+
"userAgentPatterns": [],
|
|
258
256
|
"userAgents": []
|
|
259
257
|
},
|
|
260
258
|
{
|
|
@@ -363,9 +361,7 @@
|
|
|
363
361
|
"category": "AI_ASSISTANT",
|
|
364
362
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
365
363
|
"followsRobotsTxt": false,
|
|
366
|
-
"userAgentPatterns": [
|
|
367
|
-
""
|
|
368
|
-
],
|
|
364
|
+
"userAgentPatterns": [],
|
|
369
365
|
"userAgents": []
|
|
370
366
|
},
|
|
371
367
|
{
|
|
@@ -377,9 +373,7 @@
|
|
|
377
373
|
"category": "AI_ASSISTANT",
|
|
378
374
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
379
375
|
"followsRobotsTxt": false,
|
|
380
|
-
"userAgentPatterns": [
|
|
381
|
-
""
|
|
382
|
-
],
|
|
376
|
+
"userAgentPatterns": [],
|
|
383
377
|
"userAgents": []
|
|
384
378
|
},
|
|
385
379
|
{
|
|
@@ -391,9 +385,7 @@
|
|
|
391
385
|
"category": "AI_ASSISTANT",
|
|
392
386
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
393
387
|
"followsRobotsTxt": false,
|
|
394
|
-
"userAgentPatterns": [
|
|
395
|
-
""
|
|
396
|
-
],
|
|
388
|
+
"userAgentPatterns": [],
|
|
397
389
|
"userAgents": []
|
|
398
390
|
},
|
|
399
391
|
{
|
|
@@ -405,9 +397,7 @@
|
|
|
405
397
|
"category": "AI_ASSISTANT",
|
|
406
398
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
407
399
|
"followsRobotsTxt": false,
|
|
408
|
-
"userAgentPatterns": [
|
|
409
|
-
""
|
|
410
|
-
],
|
|
400
|
+
"userAgentPatterns": [],
|
|
411
401
|
"userAgents": []
|
|
412
402
|
},
|
|
413
403
|
{
|
|
@@ -419,9 +409,7 @@
|
|
|
419
409
|
"category": "AI_ASSISTANT",
|
|
420
410
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
421
411
|
"followsRobotsTxt": false,
|
|
422
|
-
"userAgentPatterns": [
|
|
423
|
-
""
|
|
424
|
-
],
|
|
412
|
+
"userAgentPatterns": [],
|
|
425
413
|
"userAgents": []
|
|
426
414
|
},
|
|
427
415
|
{
|
|
@@ -433,9 +421,7 @@
|
|
|
433
421
|
"category": "AI_ASSISTANT",
|
|
434
422
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
435
423
|
"followsRobotsTxt": false,
|
|
436
|
-
"userAgentPatterns": [
|
|
437
|
-
""
|
|
438
|
-
],
|
|
424
|
+
"userAgentPatterns": [],
|
|
439
425
|
"userAgents": []
|
|
440
426
|
},
|
|
441
427
|
{
|
|
@@ -447,9 +433,7 @@
|
|
|
447
433
|
"category": "AI_ASSISTANT",
|
|
448
434
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
449
435
|
"followsRobotsTxt": false,
|
|
450
|
-
"userAgentPatterns": [
|
|
451
|
-
""
|
|
452
|
-
],
|
|
436
|
+
"userAgentPatterns": [],
|
|
453
437
|
"userAgents": []
|
|
454
438
|
},
|
|
455
439
|
{
|
|
@@ -461,9 +445,7 @@
|
|
|
461
445
|
"category": "AI_ASSISTANT",
|
|
462
446
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
463
447
|
"followsRobotsTxt": false,
|
|
464
|
-
"userAgentPatterns": [
|
|
465
|
-
""
|
|
466
|
-
],
|
|
448
|
+
"userAgentPatterns": [],
|
|
467
449
|
"userAgents": []
|
|
468
450
|
},
|
|
469
451
|
{
|
|
@@ -475,9 +457,7 @@
|
|
|
475
457
|
"category": "AI_ASSISTANT",
|
|
476
458
|
"description": "AgentCore Browser provides a secure, cloud-based browser that enables AI agents to interact with websites.",
|
|
477
459
|
"followsRobotsTxt": false,
|
|
478
|
-
"userAgentPatterns": [
|
|
479
|
-
""
|
|
480
|
-
],
|
|
460
|
+
"userAgentPatterns": [],
|
|
481
461
|
"userAgents": []
|
|
482
462
|
},
|
|
483
463
|
{
|
|
@@ -1201,9 +1181,7 @@
|
|
|
1201
1181
|
"category": "AI_ASSISTANT",
|
|
1202
1182
|
"description": "Browserbase helps AI use the web. Autonomously reads, writes, and performs tasks on the web with a serverless browser.",
|
|
1203
1183
|
"followsRobotsTxt": false,
|
|
1204
|
-
"userAgentPatterns": [
|
|
1205
|
-
""
|
|
1206
|
-
],
|
|
1184
|
+
"userAgentPatterns": [],
|
|
1207
1185
|
"userAgents": []
|
|
1208
1186
|
},
|
|
1209
1187
|
{
|
|
@@ -1329,9 +1307,7 @@
|
|
|
1329
1307
|
"category": "AI_ASSISTANT",
|
|
1330
1308
|
"description": "CartAI B2B rails combine execution, payments, loyalty and affiliate networks to facilitate agentic commerce",
|
|
1331
1309
|
"followsRobotsTxt": false,
|
|
1332
|
-
"userAgentPatterns": [
|
|
1333
|
-
""
|
|
1334
|
-
],
|
|
1310
|
+
"userAgentPatterns": [],
|
|
1335
1311
|
"userAgents": []
|
|
1336
1312
|
},
|
|
1337
1313
|
{
|
|
@@ -1442,9 +1418,7 @@
|
|
|
1442
1418
|
"category": "AI_ASSISTANT",
|
|
1443
1419
|
"description": "Agent that can use its own browser to perform tasks for user.",
|
|
1444
1420
|
"followsRobotsTxt": false,
|
|
1445
|
-
"userAgentPatterns": [
|
|
1446
|
-
""
|
|
1447
|
-
],
|
|
1421
|
+
"userAgentPatterns": [],
|
|
1448
1422
|
"userAgents": []
|
|
1449
1423
|
},
|
|
1450
1424
|
{
|
|
@@ -1520,9 +1494,7 @@
|
|
|
1520
1494
|
"category": "PAGE_PREVIEW",
|
|
1521
1495
|
"description": "Cloudflare Browser Rendering",
|
|
1522
1496
|
"followsRobotsTxt": false,
|
|
1523
|
-
"userAgentPatterns": [
|
|
1524
|
-
""
|
|
1525
|
-
],
|
|
1497
|
+
"userAgentPatterns": [],
|
|
1526
1498
|
"userAgents": []
|
|
1527
1499
|
},
|
|
1528
1500
|
{
|
|
@@ -1630,9 +1602,7 @@
|
|
|
1630
1602
|
"category": "SECURITY",
|
|
1631
1603
|
"description": "Cloudflare Radar URL Scanner",
|
|
1632
1604
|
"followsRobotsTxt": false,
|
|
1633
|
-
"userAgentPatterns": [
|
|
1634
|
-
""
|
|
1635
|
-
],
|
|
1605
|
+
"userAgentPatterns": [],
|
|
1636
1606
|
"userAgents": []
|
|
1637
1607
|
},
|
|
1638
1608
|
{
|
|
@@ -4066,9 +4036,7 @@
|
|
|
4066
4036
|
"category": "AI_SEARCH",
|
|
4067
4037
|
"description": "Klaviyo’s web crawler for its Kai Customer Agent feature.",
|
|
4068
4038
|
"followsRobotsTxt": false,
|
|
4069
|
-
"userAgentPatterns": [
|
|
4070
|
-
""
|
|
4071
|
-
],
|
|
4039
|
+
"userAgentPatterns": [],
|
|
4072
4040
|
"userAgents": []
|
|
4073
4041
|
},
|
|
4074
4042
|
{
|
|
@@ -4401,9 +4369,7 @@
|
|
|
4401
4369
|
"category": "AI_ASSISTANT",
|
|
4402
4370
|
"description": "Manus is the action engine that goes beyond answers to execute tasks, automate workflows, and extend your human reach.",
|
|
4403
4371
|
"followsRobotsTxt": false,
|
|
4404
|
-
"userAgentPatterns": [
|
|
4405
|
-
""
|
|
4406
|
-
],
|
|
4372
|
+
"userAgentPatterns": [],
|
|
4407
4373
|
"userAgents": []
|
|
4408
4374
|
},
|
|
4409
4375
|
{
|
|
@@ -4968,9 +4934,7 @@
|
|
|
4968
4934
|
"category": "AI_ASSISTANT",
|
|
4969
4935
|
"description": "Executes secure checkout via browser agent using user card and signed mandate.",
|
|
4970
4936
|
"followsRobotsTxt": false,
|
|
4971
|
-
"userAgentPatterns": [
|
|
4972
|
-
""
|
|
4973
|
-
],
|
|
4937
|
+
"userAgentPatterns": [],
|
|
4974
4938
|
"userAgents": []
|
|
4975
4939
|
},
|
|
4976
4940
|
{
|
|
@@ -5921,9 +5885,7 @@
|
|
|
5921
5885
|
"category": "AGGREGATOR",
|
|
5922
5886
|
"description": "Quartr uses a crawler to obtain and deliver investor relations material",
|
|
5923
5887
|
"followsRobotsTxt": false,
|
|
5924
|
-
"userAgentPatterns": [
|
|
5925
|
-
""
|
|
5926
|
-
],
|
|
5888
|
+
"userAgentPatterns": [],
|
|
5927
5889
|
"userAgents": []
|
|
5928
5890
|
},
|
|
5929
5891
|
{
|
|
@@ -8297,9 +8259,7 @@
|
|
|
8297
8259
|
"category": "MONITORING_AND_ANALYTICS",
|
|
8298
8260
|
"description": "zvelo fetches content for web categorization.",
|
|
8299
8261
|
"followsRobotsTxt": false,
|
|
8300
|
-
"userAgentPatterns": [
|
|
8301
|
-
""
|
|
8302
|
-
],
|
|
8262
|
+
"userAgentPatterns": [],
|
|
8303
8263
|
"userAgents": []
|
|
8304
8264
|
}
|
|
8305
8265
|
]
|
package/.editorconfig
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# https://editorconfig.org
|
|
2
|
-
|
|
3
|
-
root = true
|
|
4
|
-
|
|
5
|
-
[*]
|
|
6
|
-
indent_style = space
|
|
7
|
-
indent_size = 2
|
|
8
|
-
end_of_line = lf
|
|
9
|
-
charset = utf-8
|
|
10
|
-
trim_trailing_whitespace = true
|
|
11
|
-
insert_final_newline = true
|
|
12
|
-
max_line_length = 80
|
|
13
|
-
indent_brace_style = 1TBS
|
|
14
|
-
spaces_around_operators = true
|
|
15
|
-
quote_type = auto
|
|
16
|
-
|
|
17
|
-
[package.json]
|
|
18
|
-
indent_style = space
|
|
19
|
-
indent_size = 2
|
package/.gitattributes
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
* text=auto
|
package/.github/dependabot.yml
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
name: cron
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
workflow_dispatch:
|
|
5
|
-
schedule:
|
|
6
|
-
# Cron job every Monday at 00:00
|
|
7
|
-
# https://crontab.guru/every-monday
|
|
8
|
-
- cron: '0 0 * * MON'
|
|
9
|
-
|
|
10
|
-
jobs:
|
|
11
|
-
update:
|
|
12
|
-
runs-on: ubuntu-latest
|
|
13
|
-
steps:
|
|
14
|
-
- name: Checkout
|
|
15
|
-
uses: actions/checkout@v4
|
|
16
|
-
with:
|
|
17
|
-
token: ${{ secrets.GH_TOKEN }}
|
|
18
|
-
- name: Setup Node.js
|
|
19
|
-
uses: actions/setup-node@v4
|
|
20
|
-
with:
|
|
21
|
-
node-version: lts/*
|
|
22
|
-
- name: Update
|
|
23
|
-
env:
|
|
24
|
-
CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }}
|
|
25
|
-
CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
|
|
26
|
-
run: |
|
|
27
|
-
git config --global user.email ${{ secrets.GIT_EMAIL }}
|
|
28
|
-
git config --global user.name ${{ secrets.GIT_USERNAME }}
|
|
29
|
-
node scripts/fetch-bots.mjs
|
|
30
|
-
git add src/index.json
|
|
31
|
-
git diff-index --quiet HEAD || git commit -m 'build(update): bots' --no-verify
|
|
32
|
-
git push origin ${{ github.head_ref }} || true
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
name: main
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- master
|
|
7
|
-
|
|
8
|
-
jobs:
|
|
9
|
-
contributors:
|
|
10
|
-
if: "${{ github.event.head_commit.message != 'build: contributors' }}"
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
steps:
|
|
13
|
-
- name: Checkout
|
|
14
|
-
uses: actions/checkout@v6
|
|
15
|
-
with:
|
|
16
|
-
fetch-depth: 0
|
|
17
|
-
token: ${{ secrets.GITHUB_TOKEN }}
|
|
18
|
-
- name: Setup Node.js
|
|
19
|
-
uses: actions/setup-node@v6
|
|
20
|
-
with:
|
|
21
|
-
node-version: lts/*
|
|
22
|
-
- name: Contributors
|
|
23
|
-
run: |
|
|
24
|
-
git config --global user.email ${{ secrets.GIT_EMAIL }}
|
|
25
|
-
git config --global user.name ${{ secrets.GIT_USERNAME }}
|
|
26
|
-
npm run contributors
|
|
27
|
-
- name: Push changes
|
|
28
|
-
run: |
|
|
29
|
-
git push origin ${{ github.head_ref }}
|
|
30
|
-
|
|
31
|
-
release:
|
|
32
|
-
if: |
|
|
33
|
-
!startsWith(github.event.head_commit.message, 'chore(release):') &&
|
|
34
|
-
!startsWith(github.event.head_commit.message, 'docs:') &&
|
|
35
|
-
!startsWith(github.event.head_commit.message, 'ci:')
|
|
36
|
-
needs: [contributors]
|
|
37
|
-
runs-on: ubuntu-latest
|
|
38
|
-
steps:
|
|
39
|
-
- name: Checkout
|
|
40
|
-
uses: actions/checkout@v6
|
|
41
|
-
with:
|
|
42
|
-
token: ${{ secrets.GITHUB_TOKEN }}
|
|
43
|
-
- name: Setup Node.js
|
|
44
|
-
uses: actions/setup-node@v6
|
|
45
|
-
with:
|
|
46
|
-
node-version: lts/*
|
|
47
|
-
- name: Setup PNPM
|
|
48
|
-
uses: pnpm/action-setup@v4
|
|
49
|
-
with:
|
|
50
|
-
version: latest
|
|
51
|
-
run_install: true
|
|
52
|
-
- name: Test
|
|
53
|
-
run: pnpm test
|
|
54
|
-
- name: Report
|
|
55
|
-
run: npx c8 report --reporter=text-lcov > coverage/lcov.info
|
|
56
|
-
- name: Coverage
|
|
57
|
-
uses: coverallsapp/github-action@main
|
|
58
|
-
with:
|
|
59
|
-
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
60
|
-
- name: Release
|
|
61
|
-
env:
|
|
62
|
-
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
|
63
|
-
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
64
|
-
run: |
|
|
65
|
-
git config --global user.email ${{ secrets.GIT_EMAIL }}
|
|
66
|
-
git config --global user.name ${{ secrets.GIT_USERNAME }}
|
|
67
|
-
git pull origin master
|
|
68
|
-
pnpm run release
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
name: pull_request
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- master
|
|
7
|
-
pull_request:
|
|
8
|
-
branches:
|
|
9
|
-
- master
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test:
|
|
13
|
-
if: github.ref != 'refs/heads/master'
|
|
14
|
-
runs-on: ubuntu-latest
|
|
15
|
-
steps:
|
|
16
|
-
- name: Checkout
|
|
17
|
-
uses: actions/checkout@v6
|
|
18
|
-
with:
|
|
19
|
-
token: ${{ secrets.GITHUB_TOKEN }}
|
|
20
|
-
- name: Setup Node.js
|
|
21
|
-
uses: actions/setup-node@v6
|
|
22
|
-
with:
|
|
23
|
-
node-version: lts/*
|
|
24
|
-
- name: Setup PNPM
|
|
25
|
-
uses: pnpm/action-setup@v4
|
|
26
|
-
with:
|
|
27
|
-
version: latest
|
|
28
|
-
run_install: true
|
|
29
|
-
- name: Test
|
|
30
|
-
run: pnpm test
|
|
31
|
-
- name: Report
|
|
32
|
-
run: npx c8 report --reporter=text-lcov > coverage/lcov.info
|
|
33
|
-
- name: Coverage
|
|
34
|
-
uses: coverallsapp/github-action@main
|
|
35
|
-
with:
|
|
36
|
-
github-token: ${{ secrets.GITHUB_TOKEN }}
|
package/scripts/fetch-bots.mjs
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import { writeFile } from 'node:fs/promises'
|
|
2
|
-
import pMap from 'p-map'
|
|
3
|
-
|
|
4
|
-
const cloudflare = path =>
|
|
5
|
-
fetch(`https://api.cloudflare.com/client/v4/radar/${path}`, {
|
|
6
|
-
headers: {
|
|
7
|
-
'x-auth-email': process.env.CLOUDFLARE_EMAIL,
|
|
8
|
-
'x-auth-key': process.env.CLOUDFLARE_API_KEY
|
|
9
|
-
}
|
|
10
|
-
}).then(res => res.json())
|
|
11
|
-
|
|
12
|
-
const allBots = (await cloudflare('/bots?limit=999')).result.bots
|
|
13
|
-
|
|
14
|
-
let completed = 0
|
|
15
|
-
|
|
16
|
-
const percentage = completed =>
|
|
17
|
-
`${Math.round((completed / allBots.length) * 100)}%`
|
|
18
|
-
|
|
19
|
-
const bots = await pMap(
|
|
20
|
-
allBots,
|
|
21
|
-
async bot => {
|
|
22
|
-
const details = await cloudflare(`/bots/${bot.slug}`)
|
|
23
|
-
process.stdout.write(`…${percentage(++completed)}`)
|
|
24
|
-
return details.result.bot
|
|
25
|
-
},
|
|
26
|
-
{ concurrency: 2 }
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
await writeFile('src/index.json', JSON.stringify(bots, null, 2))
|