cloudflare-bot-directory 0.0.0 → 1.0.0

package/README.md

@@ -1,4 +1,4 @@
-# cloudflare-bot-directory
+# CloudFlare Bot Directory
 
 <p align="center">
   <br>
@@ -6,43 +6,70 @@
   <br>
 </p>
 
-> The CloudFlare Radar Bot Directory as a JSON. Check [index.json](/src/index.json)
+![Last version](https://img.shields.io/github/tag/kikobeats/cloudflare-bot-directory.svg?style=flat-square)
+[![NPM Status](https://img.shields.io/npm/dm/cloudflare-bot-directory.svg?style=flat-square)](https://www.npmjs.org/package/cloudflare-bot-directory)
+
+> A comprehensive list of **500+ verified bots** and web crawlers from [CloudFlare Radar](https://radar.cloudflare.com/traffic/verified-bots), available as a JSON dataset for bot detection, user agent analysis, and web scraping identification.
+
+## Why
+
+Identifying legitimate bots from malicious scrapers is essential for web security and analytics. This package provides the official CloudFlare Radar verified bots directory, helping you:
+
+- **Detect verified bots** – Identify legitimate crawlers like Googlebot, Bingbot, and more
+- **Filter analytics** – Exclude known bots from your traffic reports
+- **Allow-list crawlers** – Permit verified bots while blocking suspicious traffic
+- **User agent lookup** – Match user agent strings against known bot patterns
+
+## Data Structure
+
+Each bot entry includes:
+
+| Field               | Description                                    |
+| ------------------- | ---------------------------------------------- |
+| `slug`              | URL-friendly unique identifier                 |
+| `name`              | Human-readable bot name                        |
+| `kind`              | Bot kind classification                        |
+| `operator`          | Company or organization operating the bot      |
+| `operatorUrl`       | URL to operator's documentation                |
+| `category`          | Bot category (Search Engine, Monitoring, etc.) |
+| `description`       | What the bot does                              |
+| `followsRobotsTxt`  | Whether the bot respects robots.txt            |
+| `userAgentPatterns` | User agent pattern(s) for matching             |
+| `userAgents`        | Known user agent string(s)                     |
 
 ## Install
 
 ```bash
-$ npm install cloudflare-bot-directory --save
+npm install cloudflare-bot-directory
 ```
 
 ## Usage
 
 ```js
-const cloudflareBotDirectory = require('cloudflare-bot-directory')
-
-
-console.log(cloudflareBotDirectory.map(item => item.slug))
-
-> console.log(cloudflareBotDirectory.map(item => item.slug))
-// [
-//   '2checkout',
-//   '360monitoring',
-//   'aasa',
-//   'accessible-web-bot',
-//   'accessstatus',
-//   'acquia-optimize-monsido',
-//   'activecomply-bot',
-//   'adagiobot',
-//   'addsearchbot',
-//   'addthis',
-//   'adidxbot',
-//   'adstxtcrawler',
-//   ... 502 more items
-// ]
+const bots = require('cloudflare-bot-directory')
+
+// Get all bot slugs
+console.log(bots.map(bot => bot.slug))
+// ['2checkout', '360monitoring', 'googlebot', 'bingbot', ...]
+
+// Find a specific bot
+const googlebot = bots.find(bot => bot.slug === 'googlebot')
+
+// Filter by category
+const searchBots = bots.filter(bot => bot.category === 'Search Engine Crawler')
+
+// Check if a user agent is a known bot
+const isKnownBot = (userAgent) =>
+  bots.some(bot => userAgent.includes(bot.name))
 ```
 
+## Related
+
+- [top-user-agents](https://github.com/microlinkhq/top-user-agents) – An always up-to-date list of the top 100 HTTP user-agents most used over the Internet.
+
 ## License
 
 **cloudflare-bot-directory** © [Kiko Beats](https://kikobeats.com), released under the [MIT](https://github.com/kikobeats/cloudflare-bot-directory/blob/master/LICENSE.md) License.<br>
 Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/kikobeats/cloudflare-bot-directory/contributors).
 
-> [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/kikobeats) · Twitter [@kikobeats](https://twitter.com/kikobeats)
+> [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/kikobeats) · X [@kikobeats](https://x.com/kikobeats)

package/package.json

@@ -1,8 +1,10 @@
 {
   "name": "cloudflare-bot-directory",
-  "description": "The CloudFlare bots directory",
+  "description": "CloudFlare Radar verified bots directory - 500+ web crawlers, search engine bots, and user agents as JSON for bot detection and traffic filtering",
   "homepage": "https://github.com/kikobeats/cloudflare-bot-directory",
-  "version": "0.0.0",
+  "version": "1.0.0",
+  "types": "./src/index.d.ts",
+  "main": "./src/index.js",
   "exports": {
     ".": "./src/index.js"
   },
@@ -11,6 +13,7 @@
     "name": "Kiko Beats",
     "url": "https://kikobeats.com"
   },
+  "contributors": [],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/kikobeats/cloudflare-bot-directory.git"
@@ -19,10 +22,29 @@
     "url": "https://github.com/kikobeats/cloudflare-bot-directory/issues"
   },
   "keywords": [
+    "bingbot",
+    "bot-detection",
+    "bot-filtering",
+    "bot-list",
     "bots",
     "cloudflare",
+    "cloudflare-radar",
+    "crawler",
+    "crawlers",
     "directory",
-    "radar"
+    "googlebot",
+    "radar",
+    "scraper",
+    "scraper-detection",
+    "search-engine",
+    "security",
+    "spam-detection",
+    "traffic-filtering",
+    "user-agent",
+    "user-agents",
+    "verified-bots",
+    "web-crawler",
+    "web-security"
   ],
   "devDependencies": {
     "@commitlint/cli": "latest",
@@ -36,6 +58,7 @@
     "git-authors-cli": "latest",
     "github-generate-release": "latest",
     "nano-staged": "latest",
+    "p-map": "latest",
     "simple-git-hooks": "latest",
     "standard": "latest",
     "standard-version": "latest"
@@ -43,6 +66,9 @@
   "engines": {
     "node": ">= 20"
   },
+  "files": [
+    "src"
+  ],
   "license": "MIT",
   "commitlint": {
     "extends": [
@@ -67,9 +93,6 @@
     "commit-msg": "npx commitlint --edit",
     "pre-commit": "npx nano-staged"
   },
-  "dependencies": {
-    "p-map": "~7.0.4"
-  },
   "scripts": {
     "clean": "rm -rf node_modules",
     "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true",
@@ -78,12 +101,12 @@
     "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)",
     "pretest": "npm run lint",
     "release": "pnpm run release:version && pnpm run release:changelog && pnpm run release:commit && pnpm run release:tag",
-    "release:github": "github-generate-release",
-    "release:tags": "git push origin HEAD:master --follow-tags",
-    "test": "c8 ava",
     "release:changelog": "conventional-changelog -p conventionalcommits -i CHANGELOG.md -s",
     "release:commit": "git add package.json CHANGELOG.md && git commit -m \"chore(release): $(node -p \"require('./package.json').version\")\"",
+    "release:github": "github-generate-release",
     "release:tag": "git tag -a v$(node -p \"require('./package.json').version\") -m \"v$(node -p \"require('./package.json').version\")\"",
-    "release:version": "standard-version --skip.changelog --skip.commit --skip.tag"
+    "release:tags": "git push origin HEAD:master --follow-tags",
+    "release:version": "standard-version --skip.changelog --skip.commit --skip.tag",
+    "test": "c8 ava"
   }
 }

package/src/index.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Bot category types from CloudFlare Radar
+ */
+export type BotCategory =
+  | 'ACCESSIBILITY'
+  | 'ADVERTISING'
+  | 'AI_CRAWLER'
+  | 'ARCHIVING'
+  | 'FEED_FETCHER'
+  | 'MONITORING_AND_ANALYTICS'
+  | 'ONLINE_PAYMENTS'
+  | 'OTHER'
+  | 'PAGE_PREVIEW'
+  | 'SEARCH_ENGINE_CRAWLER'
+  | 'SECURITY'
+  | 'SEO'
+  | 'SOCIAL_MEDIA_MARKETING'
+  | 'WEBHOOKS'
+
+/**
+ * Bot kind types
+ */
+export type BotKind = 'BOT'
+
+/**
+ * A verified bot entry from CloudFlare Radar Bot Directory
+ */
+export interface Bot {
+  /** URL-friendly unique identifier */
+  slug: string
+  /** Human-readable bot name */
+  name: string
+  /** Bot kind classification */
+  kind: BotKind
+  /** Company or organization operating the bot */
+  operator: string
+  /** URL to operator's documentation or website */
+  operatorUrl: string
+  /** Bot category classification */
+  category: BotCategory
+  /** Description of what the bot does */
+  description: string
+  /** Whether the bot respects robots.txt */
+  followsRobotsTxt: boolean
+  /** User agent pattern(s) for matching */
+  userAgentPatterns: string[]
+  /** Known user agent string(s) */
+  userAgents: string[]
+}
+
+/**
+ * Array of all verified bots from CloudFlare Radar Bot Directory
+ */
+declare const bots: Bot[]
+
+export default bots

package/.editorconfig

@@ -1,19 +0,0 @@
-# https://editorconfig.org
-
-root = true
-
-[*]
-indent_style = space
-indent_size = 2
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-max_line_length = 80
-indent_brace_style = 1TBS
-spaces_around_operators = true
-quote_type = auto
-
-[package.json]
-indent_style = space
-indent_size = 2

package/.gitattributes

@@ -1 +0,0 @@
-* text=auto

package/.github/dependabot.yml

@@ -1,11 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: npm
-    directory: '/'
-    schedule:
-      interval: daily
-  - package-ecosystem: 'github-actions'
-    directory: '/'
-    schedule:
-      # Check for updates to GitHub Actions every weekday
-      interval: 'daily'

package/.github/workflows/cron.yml

@@ -1,32 +0,0 @@
-name: cron
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Cron job every Monday at 00:00
-    # https://crontab.guru/every-monday
-    - cron: '0 0 * * MON'
-
-jobs:
-  update:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GH_TOKEN }}
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-      - name: Update
-        env:
-          CLOUDFLARE_EMAIL: ${{ secrets.CLOUDFLARE_EMAIL }}
-          CLOUDFLARE_API_KEY: ${{ secrets.CLOUDFLARE_API_KEY }}
-        run: |
-          git config --global user.email ${{ secrets.GIT_EMAIL }}
-          git config --global user.name ${{ secrets.GIT_USERNAME }}
-          node scripts/fetch-bots.mjs
-          git add src/index.json
-          git diff-index --quiet HEAD || git commit -m 'build(update): bots' --no-verify
-          git push origin ${{ github.head_ref }} || true

package/.github/workflows/main.yml

@@ -1,68 +0,0 @@
-name: main
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  contributors:
-    if: "${{ github.event.head_commit.message != 'build: contributors' }}"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: lts/*
-      - name: Contributors
-        run: |
-          git config --global user.email ${{ secrets.GIT_EMAIL }}
-          git config --global user.name ${{ secrets.GIT_USERNAME }}
-          npm run contributors
-      - name: Push changes
-        run: |
-          git push origin ${{ github.head_ref }}
-
-  release:
-    if: |
-      !startsWith(github.event.head_commit.message, 'chore(release):') &&
-      !startsWith(github.event.head_commit.message, 'docs:') &&
-      !startsWith(github.event.head_commit.message, 'ci:')
-    needs: [contributors]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: lts/*
-      - name: Setup PNPM
-        uses: pnpm/action-setup@v4
-        with:
-          version: latest
-          run_install: true
-      - name: Test
-        run: pnpm test
-      - name: Report
-        run: npx c8 report --reporter=text-lcov > coverage/lcov.info
-      - name: Coverage
-        uses: coverallsapp/github-action@main
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Release
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          git config --global user.email ${{ secrets.GIT_EMAIL }}
-          git config --global user.name ${{ secrets.GIT_USERNAME }}
-          git pull origin master
-          pnpm run release

package/.github/workflows/pull_request.yml

@@ -1,36 +0,0 @@
-name: pull_request
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  test:
-    if: github.ref != 'refs/heads/master'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: lts/*
-      - name: Setup PNPM
-        uses: pnpm/action-setup@v4
-        with:
-          version: latest
-          run_install: true
-      - name: Test
-        run: pnpm test
-      - name: Report
-        run: npx c8 report --reporter=text-lcov > coverage/lcov.info
-      - name: Coverage
-        uses: coverallsapp/github-action@main
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

package/scripts/fetch-bots.mjs

@@ -1,29 +0,0 @@
-import { writeFile } from 'node:fs/promises'
-import pMap from 'p-map'
-
-const cloudflare = path =>
-  fetch(`https://api.cloudflare.com/client/v4/radar/${path}`, {
-    headers: {
-      'x-auth-email': process.env.CLOUDFLARE_EMAIL,
-      'x-auth-key': process.env.CLOUDFLARE_API_KEY
-    }
-  }).then(res => res.json())
-
-const allBots = (await cloudflare('/bots?limit=999')).result.bots
-
-let completed = 0
-
-const percentage = completed =>
-  `${Math.round((completed / allBots.length) * 100)}%`
-
-const bots = await pMap(
-  allBots,
-  async bot => {
-    const details = await cloudflare(`/bots/${bot.slug}`)
-    process.stdout.write(`…${percentage(++completed)}`)
-    return details.result.bot
-  },
-  { concurrency: 2 }
-)
-
-await writeFile('src/index.json', JSON.stringify(bots, null, 2))