bot_verification 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b20a18dec9de1772a0c48135b190d1018b823a418ae5d1e114210dfe53fc6e5b
+  data.tar.gz: 4178233855e6c63a40b4a3bca25215b000888068805855f85a20d159681f7032
+SHA512:
+  metadata.gz: 5540546cf2c83a714426c1fce497d7e76f753bfc6640d6604311afacea228dba21689beb1422bbf15d900f943fc79a72375b5d2ae0c161b149744ebd4a5b629d
+  data.tar.gz: 3089f729478e5d40ffafa31582670f6d83e34bcd11df3d1ed969d06cb8a83f87208078019cb622a5d86c849a817fe354cac2acc3984a60d1070a0c1f084eb122

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2025-01-19
+
+### Added
+- Initial release
+- IP range verification for search engine bots (Google, Bing)
+- IP range verification for AI bots (OpenAI GPTBot, ChatGPT-User, OAI-SearchBot, PerplexityBot, Amazonbot)
+- Reverse DNS verification fallback for search engines (Google, Bing, Apple, Yandex, Baidu)
+- User agent pattern detection for search, AI, and social bots
+- Rails controller concern with session-based caching
+- Configurable caching (Rails cache and session)
+- Background job for IP range refresh
+- Rake tasks for management and debugging
+- Rails generator for installation

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -0,0 +1,256 @@
+PATH
+  remote: .
+  specs:
+    bot_verification (0.1.0)
+      rails (>= 7.0)
+      resolv (>= 0.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    action_text-trix (2.1.16)
+      railties
+    actioncable (8.1.2)
+      actionpack (= 8.1.2)
+      activesupport (= 8.1.2)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+      zeitwerk (~> 2.6)
+    actionmailbox (8.1.2)
+      actionpack (= 8.1.2)
+      activejob (= 8.1.2)
+      activerecord (= 8.1.2)
+      activestorage (= 8.1.2)
+      activesupport (= 8.1.2)
+      mail (>= 2.8.0)
+    actionmailer (8.1.2)
+      actionpack (= 8.1.2)
+      actionview (= 8.1.2)
+      activejob (= 8.1.2)
+      activesupport (= 8.1.2)
+      mail (>= 2.8.0)
+      rails-dom-testing (~> 2.2)
+    actionpack (8.1.2)
+      actionview (= 8.1.2)
+      activesupport (= 8.1.2)
+      nokogiri (>= 1.8.5)
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+      useragent (~> 0.16)
+    actiontext (8.1.2)
+      action_text-trix (~> 2.1.15)
+      actionpack (= 8.1.2)
+      activerecord (= 8.1.2)
+      activestorage (= 8.1.2)
+      activesupport (= 8.1.2)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (8.1.2)
+      activesupport (= 8.1.2)
+      builder (~> 3.1)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activejob (8.1.2)
+      activesupport (= 8.1.2)
+      globalid (>= 0.3.6)
+    activemodel (8.1.2)
+      activesupport (= 8.1.2)
+    activerecord (8.1.2)
+      activemodel (= 8.1.2)
+      activesupport (= 8.1.2)
+      timeout (>= 0.4.0)
+    activestorage (8.1.2)
+      actionpack (= 8.1.2)
+      activejob (= 8.1.2)
+      activerecord (= 8.1.2)
+      activesupport (= 8.1.2)
+      marcel (~> 1.0)
+    activesupport (8.1.2)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      json
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+    base64 (0.3.0)
+    bigdecimal (4.0.1)
+    builder (3.3.0)
+    concurrent-ruby (1.3.6)
+    connection_pool (3.0.2)
+    crass (1.0.6)
+    date (3.5.1)
+    diff-lcs (1.6.2)
+    drb (2.2.3)
+    erb (6.0.1)
+    erubi (1.13.1)
+    globalid (1.3.0)
+      activesupport (>= 6.1)
+    i18n (1.14.8)
+      concurrent-ruby (~> 1.0)
+    io-console (0.8.2)
+    irb (1.16.0)
+      pp (>= 0.6.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    json (2.18.0)
+    logger (1.7.0)
+    loofah (2.25.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.9.0)
+      logger
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.1.0)
+    mini_mime (1.1.5)
+    minitest (6.0.1)
+      prism (~> 1.5)
+    net-imap (0.6.2)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.5.1)
+      net-protocol
+    nio4r (2.7.5)
+    nokogiri (1.19.0-aarch64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-aarch64-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm-linux-musl)
+      racc (~> 1.4)
+    nokogiri (1.19.0-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-linux-gnu)
+      racc (~> 1.4)
+    nokogiri (1.19.0-x86_64-linux-musl)
+      racc (~> 1.4)
+    pp (0.6.3)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.8.0)
+    psych (5.3.1)
+      date
+      stringio
+    racc (1.8.1)
+    rack (3.2.4)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
+    rack-test (2.2.0)
+      rack (>= 1.3)
+    rackup (2.3.1)
+      rack (>= 3)
+    rails (8.1.2)
+      actioncable (= 8.1.2)
+      actionmailbox (= 8.1.2)
+      actionmailer (= 8.1.2)
+      actionpack (= 8.1.2)
+      actiontext (= 8.1.2)
+      actionview (= 8.1.2)
+      activejob (= 8.1.2)
+      activemodel (= 8.1.2)
+      activerecord (= 8.1.2)
+      activestorage (= 8.1.2)
+      activesupport (= 8.1.2)
+      bundler (>= 1.15.0)
+      railties (= 8.1.2)
+    rails-dom-testing (2.3.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.2)
+      loofah (~> 2.21)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    railties (8.1.2)
+      actionpack (= 8.1.2)
+      activesupport (= 8.1.2)
+      irb (~> 1.13)
+      rackup (>= 1.0.0)
+      rake (>= 12.2)
+      thor (~> 1.0, >= 1.2.2)
+      tsort (>= 0.2)
+      zeitwerk (~> 2.6)
+    rake (13.3.1)
+    rdoc (7.1.0)
+      erb
+      psych (>= 4.0.0)
+      tsort
+    reline (0.6.3)
+      io-console (~> 0.5)
+    resolv (0.7.0)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.7)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.6)
+    securerandom (0.4.1)
+    sqlite3 (2.9.0-aarch64-linux-gnu)
+    sqlite3 (2.9.0-aarch64-linux-musl)
+    sqlite3 (2.9.0-arm-linux-gnu)
+    sqlite3 (2.9.0-arm-linux-musl)
+    sqlite3 (2.9.0-arm64-darwin)
+    sqlite3 (2.9.0-x86_64-darwin)
+    sqlite3 (2.9.0-x86_64-linux-gnu)
+    sqlite3 (2.9.0-x86_64-linux-musl)
+    stringio (3.2.0)
+    thor (1.5.0)
+    timeout (0.6.0)
+    tsort (0.2.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.1.1)
+    useragent (0.16.11)
+    websocket-driver (0.8.0)
+      base64
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.7.4)
+
+PLATFORMS
+  aarch64-linux
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x86_64-darwin
+  x86_64-linux
+  x86_64-linux-gnu
+  x86_64-linux-musl
+
+DEPENDENCIES
+  bot_verification!
+  rake (~> 13.0)
+  rspec (~> 3.12)
+  sqlite3 (>= 2.1)
+
+BUNDLED WITH
+   2.5.23

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Web Ventures Ltd - www.webven.nz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,355 @@
+# BotVerification
+
+[![Gem Version](https://badge.fury.io/rb/bot_verification.svg)](https://badge.fury.io/rb/bot_verification)
+[![CI](https://github.com/webventures/bot_verification/actions/workflows/ci.yml/badge.svg)](https://github.com/webventures/bot_verification/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A Rails gem for verifying that requests claiming to be from search engine bots (Google, Bing, etc.) and AI bots (GPTBot, PerplexityBot) are actually from those services.
+
+## Why?
+
+User agents can be easily spoofed. This gem verifies bot requests using:
+
+1. **IP Range Matching** (fast) - Checks against known IP ranges from official sources
+2. **Reverse DNS Verification** (authoritative) - Falls back to DNS verification if IP range check fails
+
+## Supported Bots
+
+### Search Engines (IP + DNS verification)
+- Google (Googlebot, Google-Extended, etc.)
+- Bing (Bingbot, BingPreview)
+- Apple (Applebot)
+- Yandex (YandexBot)
+- Baidu (Baiduspider)
+
+### AI Bots (IP verification only)
+- OpenAI (GPTBot, ChatGPT-User, OAI-SearchBot)
+- Perplexity (PerplexityBot)
+- Amazon (Amazonbot)
+
+### Social Bots (user agent only - no verification available)
+- Facebook, Twitter, LinkedIn, Slack, Discord, Telegram
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "bot_verification"
+```
+
+Run the installer:
+
+```bash
+bundle install
+rails generate bot_verification:install
+rails db:migrate
+```
+
+Fetch initial IP ranges:
+
+```bash
+rails bot_verification:refresh
+```
+
+## Usage
+
+### In Controllers
+
+```ruby
+class MyController < ApplicationController
+  include BotVerification::ControllerConcern
+
+  def show
+    if verified_good_bot?
+      # Request is from a verified search engine bot
+      # Serve full content
+    elsif verified_good_bot?(mode: :search_and_ai)
+      # Also includes verified AI bots (GPTBot, etc.)
+    else
+      # Regular user or unverified bot
+      # Apply rate limiting, require auth, etc.
+    end
+  end
+end
+```
+
+### Verification Modes
+
+```ruby
+# Only verified search engine bots (default, most secure)
+verified_good_bot?(mode: :search_engines)
+
+# Search engines + verified AI bots
+verified_good_bot?(mode: :search_and_ai)
+
+# All known bot patterns (trusts user agent, least secure)
+verified_good_bot?(mode: :all_known)
+```
+
+### Check Specifically for AI Bots
+
+```ruby
+if verified_ai_bot?
+  # Verified AI bot (GPTBot, PerplexityBot, etc.)
+end
+
+# Include unverifiable AI bots like ClaudeBot
+if verified_ai_bot?(strict: false)
+  # Any recognized AI bot
+end
+```
+
+### Direct Service Usage
+
+```ruby
+# Verify a request
+BotVerification.verify(ip, user_agent)
+BotVerification.verify(ip, user_agent, mode: :search_and_ai)
+
+# Check specific IP
+BotVerification.verify_ip("66.249.66.1", :google)
+
+# Detect bot type from user agent
+BotVerification.detect_bot("Googlebot/2.1") # => :google
+```
+
+## Performance
+
+Verification uses a tiered approach, fastest first:
+
+| Tier | Method | Latency | When Used |
+|------|--------|---------|-----------|
+| 1 | Session cache | ~1ms | Same session, same IP+UA |
+| 2 | Rails cache | ~1ms | Previously verified IP |
+| 3 | IP range check | ~5-10ms | First verification for IP |
+| 4 | Reverse DNS | 100-2000ms | IP range miss, only for bots |
+
+**Important:** DNS lookups only occur when:
+- User agent matches a known bot pattern, AND
+- IP range check fails
+
+Regular users never trigger DNS lookups.
+
+## Configuration
+
+```ruby
+# config/initializers/bot_verification.rb
+
+BotVerification.configure do |config|
+  # Table name for storing bot IP ranges
+  config.table_name = "bot_ip_ranges"
+
+  # Skip DNS verification entirely (only use IP range matching)
+  # Set to true if DNS lookups are unacceptable for your project.
+  # Note: Apple, Yandex, Baidu don't publish IP ranges, so they won't
+  # be verifiable when DNS is skipped.
+  config.skip_dns_verification = false
+
+  # Timeout for each DNS lookup (seconds)
+  # Only applies when skip_dns_verification is false
+  config.dns_timeout = 1.0
+
+  # Total timeout for all DNS operations (seconds)
+  # Only applies when skip_dns_verification is false
+  config.dns_total_timeout = 2.0
+
+  # How long to cache verification results
+  config.cache_ttl = 24.hours
+
+  # How long to cache in session
+  config.session_cache_ttl = 1.hour
+
+  # Custom model class (optional)
+  # config.ip_range_model_name = "MyBotIpRange"
+
+  # Error callback - integrate with error tracking (Airbrake, Sentry, etc.)
+  config.on_error = ->(error, context) {
+    Airbrake.notify(error, context)
+    # or: Sentry.capture_exception(error, extra: context)
+  }
+
+  # Refresh complete callback - for monitoring/notifications
+  config.on_refresh_complete = ->(results) {
+    failures = results.select { |_, r| !r[:success] }
+    if failures.any?
+      Rails.logger.warn("BotVerification refresh failures: #{failures.keys}")
+    end
+  }
+end
+```
+
+## Error Handling
+
+The gem reports errors through multiple channels:
+
+### Logger
+
+All operations log to `config.logger` (defaults to `Rails.logger`):
+- `INFO` - Successful operations
+- `WARN` - Non-critical issues (e.g., no ranges fetched)
+- `ERROR` - Failures (HTTP errors, parse errors)
+
+### Error Callback
+
+For integration with error tracking services:
+
+```ruby
+config.on_error = ->(error, context) {
+  # error: The exception object
+  # context: Hash with :bot_type, :source, :url
+  Airbrake.notify(error, context)
+}
+```
+
+### Refresh Results
+
+`refresh_ip_ranges!` returns a hash with success/failure for each bot type:
+
+```ruby
+results = BotVerification.refresh_ip_ranges!
+# => {
+#   google: { success: true, count: 142 },
+#   bing: { success: true, count: 8 },
+#   openai_gptbot: { success: false, error: "HTTP 503: Service Unavailable" }
+# }
+
+# Check for failures
+failures = results.select { |_, r| !r[:success] }
+```
+
+## Refreshing IP Ranges
+
+IP ranges should be refreshed daily. Choose the method that fits your deployment:
+
+### Rake Task (Heroku Scheduler, cron, etc.)
+
+```bash
+# Refresh all bot types
+rails bot_verification:refresh
+
+# Refresh specific bot type
+rails bot_verification:refresh_bot[google]
+```
+
+**Heroku Scheduler:** Add `rake bot_verification:refresh` as a daily job.
+
+### Background Job (Sidekiq, etc.)
+
+The gem includes `BotVerification::RefreshJob` for background processing:
+
+```ruby
+# Enqueue to run now
+BotVerification::RefreshJob.perform_later
+
+# Refresh specific bot type
+BotVerification::RefreshJob.perform_later("google")
+
+# With Sidekiq-Cron (config/sidekiq.yml)
+:schedule:
+  refresh_bot_ips:
+    cron: '0 4 * * *'  # 4am daily
+    class: BotVerification::RefreshJob
+```
+
+### Custom Job
+
+Subclass for custom queue, error handling, or notifications:
+
+```ruby
+# app/jobs/refresh_bot_ip_ranges_job.rb
+class RefreshBotIpRangesJob < BotVerification::RefreshJob
+  queue_as :low
+
+  def perform(bot_type = nil)
+    super
+  rescue => e
+    Airbrake.notify(e)
+    raise
+  end
+end
+```
+
+### Direct Call
+
+```ruby
+# In a script or console
+BotVerification.refresh_ip_ranges!
+BotVerification.refresh_ip_ranges!(:google)
+```
+
+### Cron (via whenever gem)
+
+```ruby
+# config/schedule.rb
+every 1.day, at: '4:00 am' do
+  rake "bot_verification:refresh"
+end
+```
+
+## Rake Tasks
+
+```bash
+# Refresh all IP ranges
+rails bot_verification:refresh
+
+# Refresh specific bot type
+rails bot_verification:refresh_bot[google]
+
+# Show statistics
+rails bot_verification:stats
+
+# Clear caches
+rails bot_verification:clear_cache
+
+# Verify an IP
+rails bot_verification:verify_ip[66.249.66.1,google]
+
+# Check if table exists
+rails bot_verification:check_table
+```
+
+## Using Your Own Model
+
+If you want more control, you can use your own model:
+
+```ruby
+# app/models/my_bot_ip_range.rb
+class MyBotIpRange < ApplicationRecord
+  include BotVerification::IpRangeModel
+
+  self.table_name = "my_bot_ip_ranges"
+
+  # Add custom methods...
+end
+
+# config/initializers/bot_verification.rb
+BotVerification.configure do |config|
+  config.ip_range_model_name = "MyBotIpRange"
+end
+```
+
+## References
+
+- [Google Bot Verification](https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot)
+- [Bing Bot Verification](https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26)
+- [OpenAI Bot Documentation](https://platform.openai.com/docs/bots/)
+- [Perplexity Crawlers](https://docs.perplexity.ai/guides/bots)
+
+## Changelog
+
+For a detailed list of changes for each version of this project, please see the [CHANGELOG](CHANGELOG.md).
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/webventures/bot_verification.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](LICENSE).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec