bot_verification 0.1.0

data/lib/bot_verification/ip_range_model.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module BotVerification
+  # Module to be included in your ApplicationRecord model
+  # Provides IP range storage and lookup functionality
+  #
+  # @example Create your own model
+  #   class BotIpRange < ApplicationRecord
+  #     include BotVerification::IpRangeModel
+  #   end
+  #
+  module IpRangeModel
+    extend ActiveSupport::Concern
+
+    included do
+      validates :bot_type, presence: true, inclusion: { in: ->(_) { BotPatterns::VALID_BOT_TYPES } }
+      validates :cidr, presence: true
+      validates :ip_version, presence: true, inclusion: { in: [ 4, 6 ] }
+
+      scope :for_bot, ->(bot_type) { where(bot_type: bot_type) }
+      scope :ipv4, -> { where(ip_version: 4) }
+      scope :ipv6, -> { where(ip_version: 6) }
+    end
+
+    class_methods do
+      # Check if an IP address belongs to a known bot's IP range
+      #
+      # @param ip [String] IP address to check
+      # @param bot_type [String, Symbol] Bot type (google, bing, etc.)
+      # @return [Boolean]
+      def ip_belongs_to_bot?(ip, bot_type)
+        return false if ip.blank? || bot_type.blank?
+
+        bot_type = bot_type.to_s
+        return false unless BotPatterns::VALID_BOT_TYPES.include?(bot_type)
+
+        ranges = cached_ranges_for(bot_type)
+        return false if ranges.empty?
+
+        begin
+          ip_addr = IPAddr.new(ip)
+          ranges.any? { |range| range.include?(ip_addr) }
+        rescue IPAddr::InvalidAddressError
+          false
+        end
+      end
+
+      # Get cached IPAddr objects for a bot type
+      def cached_ranges_for(bot_type)
+        cache_key = "#{BotVerification.configuration.cache_key_prefix}:ranges:#{bot_type}:v1"
+
+        BotVerification.configuration.cache.fetch(cache_key, expires_in: 1.hour) do
+          for_bot(bot_type).pluck(:cidr).filter_map do |cidr|
+            IPAddr.new(cidr)
+          rescue IPAddr::InvalidAddressError
+            nil
+          end
+        end
+      end
+
+      # Clear the cache for a bot type
+      def clear_cache_for(bot_type)
+        cache_key = "#{BotVerification.configuration.cache_key_prefix}:ranges:#{bot_type}:v1"
+        BotVerification.configuration.cache.delete(cache_key)
+      end
+
+      # Clear all bot IP range caches
+      def clear_all_caches
+        BotPatterns::VALID_BOT_TYPES.each { |type| clear_cache_for(type) }
+      end
+
+      # Bulk import ranges for a bot type (replaces existing)
+      #
+      # @param bot_type [String] Bot type
+      # @param ranges [Array<Hash>] Array of { cidr:, ip_version:, source_url: }
+      # @return [Integer] Number of records imported
+      def import_ranges(bot_type, ranges, source_url: nil)
+        transaction do
+          for_bot(bot_type).delete_all
+
+          now = Time.current
+          records = ranges.map do |range|
+            {
+              bot_type: bot_type,
+              cidr: range[:cidr],
+              ip_version: range[:ip_version] || (range[:cidr].include?(":") ? 6 : 4),
+              source_url: range[:source_url] || source_url,
+              last_verified_at: now,
+              created_at: now,
+              updated_at: now
+            }
+          end
+
+          insert_all(records) if records.any?
+          clear_cache_for(bot_type)
+          records.size
+        end
+      end
+
+      # Check if we have any ranges for a bot type
+      def has_ranges_for?(bot_type)
+        for_bot(bot_type).exists?
+      end
+
+      # Get stats about stored ranges
+      def stats
+        BotPatterns::VALID_BOT_TYPES.index_with do |bot_type|
+          {
+            count: for_bot(bot_type).count,
+            last_updated: for_bot(bot_type).maximum(:last_verified_at)
+          }
+        end
+      end
+    end
+  end
+
+  # Default model class that uses the configurable table name
+  # This is used if you don't create your own model
+  class IpRange < ActiveRecord::Base
+    include IpRangeModel
+
+    self.table_name = -> { BotVerification.configuration.table_name }
+
+    def self.table_name
+      if @table_name.respond_to?(:call)
+        @table_name.call
+      else
+        @table_name || BotVerification.configuration.table_name
+      end
+    end
+  end
+end

data/lib/bot_verification/railtie.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module BotVerification
+  class Railtie < Rails::Railtie
+    railtie_name :bot_verification
+
+    # Load rake tasks
+    rake_tasks do
+      load File.expand_path("../tasks/bot_verification.rake", __dir__)
+    end
+
+    # Set up logger after Rails initializes
+    initializer "bot_verification.configure_logger" do
+      BotVerification.configuration.logger ||= Rails.logger
+    end
+
+    # Add generators path
+    generators do
+      require_relative "../generators/bot_verification/install_generator"
+    end
+  end
+end

data/lib/bot_verification/refresh_job.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module BotVerification
+  # Background job for refreshing bot IP ranges
+  #
+  # Can be used directly or subclassed for customization:
+  #
+  # @example Direct usage with Sidekiq
+  #   BotVerification::RefreshJob.perform_async
+  #   BotVerification::RefreshJob.perform_async("google")
+  #
+  # @example Direct usage with ActiveJob
+  #   BotVerification::RefreshJob.perform_later
+  #   BotVerification::RefreshJob.perform_later("openai_gptbot")
+  #
+  # @example Subclass for custom queue or error handling
+  #   class RefreshBotIpRangesJob < BotVerification::RefreshJob
+  #     queue_as :low
+  #
+  #     def perform(bot_type = nil)
+  #       super
+  #     rescue => e
+  #       Airbrake.notify(e)
+  #       raise
+  #     end
+  #   end
+  #
+  class RefreshJob < ActiveJob::Base
+    queue_as :default
+
+    # @param bot_type [String, Symbol, nil] Specific bot type to refresh, or nil for all
+    def perform(bot_type = nil)
+      BotVerification.refresh_ip_ranges!(bot_type)
+    end
+  end
+end

data/lib/bot_verification/service.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module BotVerification
+  # Core verification service
+  #
+  # Performance characteristics:
+  # 1. Rails cache lookup (~1ms) - checks if IP was recently verified
+  # 2. IP range check (~5-10ms) - database lookup with in-memory caching
+  # 3. Reverse DNS (~100-2000ms) - only if IP range check fails, with strict timeout
+  #
+  class Service
+    include BotPatterns
+
+    MODES = %i[search_engines search_and_ai all_known].freeze
+
+    class << self
+      # Main entry point - checks if request is from a verified good bot
+      #
+      # @param ip [String] The IP address to verify
+      # @param user_agent [String] The user agent string
+      # @param mode [Symbol] Verification mode:
+      #   - :search_engines (default) - Only IP-verified search engine bots
+      #   - :search_and_ai - Search engines + IP-verified AI bots
+      #   - :all_known - All known bot patterns (trusts user agent, least secure)
+      # @return [Boolean] true if verified good bot
+      def verified_good_bot?(ip, user_agent, mode: :search_engines)
+        return false if ip.blank? || user_agent.blank?
+        raise ArgumentError, "Invalid mode: #{mode}" unless MODES.include?(mode)
+
+        bot_type = detect_bot_type(user_agent)
+        return report_verification_result(nil, false, mode, ip, user_agent) unless bot_type
+
+        verified = case mode
+        when :search_engines
+          verify_search_engine_bot(ip, bot_type)
+        when :search_and_ai
+          verify_search_engine_bot(ip, bot_type) || verify_ai_bot_internal(ip, bot_type)
+        when :all_known
+          true
+        end
+
+        report_verification_result(bot_type, verified, mode, ip, user_agent)
+      end
+
+      # Check specifically if request is from a verified AI bot
+      #
+      # @param ip [String] The IP address to verify
+      # @param user_agent [String] The user agent string
+      # @param strict [Boolean] If true (default), require IP verification for bots that publish ranges.
+      #   Bots without published IP ranges (like Anthropic) are always trusted by user agent.
+      # @return [Boolean] true if verified AI bot
+      def verified_ai_bot?(ip, user_agent, strict: true)
+        return false if ip.blank? || user_agent.blank?
+
+        bot_type = detect_ai_bot_type(user_agent)
+        return false unless bot_type
+
+        # AI bots without published IP ranges must be trusted by user agent
+        # (there's no other way to verify them)
+        if BotPatterns::AI_BOTS_WITHOUT_VERIFICATION.include?(bot_type)
+          true
+        elsif BotPatterns::AI_BOTS_WITH_IP_RANGES.include?(bot_type)
+          verified_bot_ip?(ip, bot_type)
+        elsif strict
+          false
+        else
+          true
+        end
+      end
+
+      # Detect which bot type based on user agent (all bot types)
+      def detect_bot_type(user_agent)
+        return nil if user_agent.blank?
+
+        BotPatterns::ALL_BOT_PATTERNS.each do |bot_type, pattern|
+          return bot_type if user_agent.match?(pattern)
+        end
+        nil
+      end
+
+      # Detect AI bot type specifically
+      def detect_ai_bot_type(user_agent)
+        BotPatterns::AI_BOT_PATTERNS.each do |bot_type, pattern|
+          return bot_type if user_agent.match?(pattern)
+        end
+        nil
+      end
+
+      # Detect search engine bot type specifically
+      def detect_search_bot_type(user_agent)
+        BotPatterns::SEARCH_BOT_PATTERNS.each do |bot_type, pattern|
+          return bot_type if user_agent.match?(pattern)
+        end
+        nil
+      end
+
+      # Check if IP belongs to a known bot via IP ranges or DNS
+      def verified_bot_ip?(ip, bot_type)
+        cache_key = "#{config.cache_key_prefix}:#{bot_type}:#{ip}"
+
+        cached = cache.read(cache_key)
+        return cached unless cached.nil?
+
+        result = ip_in_known_ranges?(ip, bot_type)
+
+        # Fall back to DNS verification if IP range check failed and DNS is enabled
+        if result == false && !config.skip_dns_verification && BotPatterns::SEARCH_BOTS_WITH_DNS.include?(bot_type)
+          result = verify_by_reverse_dns_with_timeout(ip, bot_type)
+        end
+
+        cache.write(cache_key, result, expires_in: config.cache_ttl)
+        result
+      end
+
+      # Check if IP is in known IP ranges for the bot type
+      def ip_in_known_ranges?(ip, bot_type)
+        return false unless BotPatterns::ALL_BOTS_WITH_IP_RANGES.include?(bot_type)
+
+        BotVerification.ip_range_model.ip_belongs_to_bot?(ip, bot_type)
+      end
+
+      # Verify bot by reverse DNS lookup with strict timeout
+      def verify_by_reverse_dns_with_timeout(ip, bot_type)
+        Timeout.timeout(config.dns_total_timeout) do
+          verify_by_reverse_dns(ip, bot_type)
+        end
+      rescue Timeout::Error
+        config.logger.warn("BotVerification: DNS verification timed out for #{ip} (#{bot_type})")
+        false
+      end
+
+      # Verify bot by reverse DNS lookup
+      def verify_by_reverse_dns(ip, bot_type)
+        suffixes = BotPatterns::SEARCH_BOT_DNS_SUFFIXES[bot_type]
+        return false unless suffixes
+
+        hostname = reverse_dns_lookup(ip)
+        return false unless hostname
+
+        valid_suffix = suffixes.any? { |suffix| hostname.end_with?(suffix) }
+        return false unless valid_suffix
+
+        forward_ips = forward_dns_lookup(hostname)
+        forward_ips.include?(ip)
+      rescue StandardError => e
+        config.logger.warn("BotVerification: DNS verification failed for #{ip}: #{e.message}")
+        false
+      end
+
+      # Check if a user agent is a known AI bot
+      def ai_bot_user_agent?(user_agent)
+        return false if user_agent.blank?
+
+        BotPatterns::AI_BOT_PATTERNS.values.any? { |pattern| user_agent.match?(pattern) }
+      end
+
+      # Check if a user agent is a known search engine bot
+      def search_bot_user_agent?(user_agent)
+        return false if user_agent.blank?
+
+        BotPatterns::SEARCH_BOT_PATTERNS.values.any? { |pattern| user_agent.match?(pattern) }
+      end
+
+      # Generate a cache key for session-based caching
+      def session_cache_key(ip, user_agent)
+        fingerprint = Digest::SHA256.hexdigest("#{ip}:#{user_agent}")[0, 16]
+        "bot_verify:#{fingerprint}"
+      end
+
+      private
+
+      def verify_search_engine_bot(ip, bot_type)
+        return false unless BotPatterns::SEARCH_BOT_PATTERNS.key?(bot_type)
+        return false unless BotPatterns::SEARCH_BOTS_WITH_DNS.include?(bot_type)
+
+        verified_bot_ip?(ip, bot_type)
+      end
+
+      def verify_ai_bot_internal(ip, bot_type)
+        return false unless BotPatterns::AI_BOT_PATTERNS.key?(bot_type)
+
+        # AI bots without published IP ranges are trusted by user agent
+        if BotPatterns::AI_BOTS_WITHOUT_VERIFICATION.include?(bot_type)
+          true
+        elsif BotPatterns::AI_BOTS_WITH_IP_RANGES.include?(bot_type)
+          verified_bot_ip?(ip, bot_type)
+        else
+          false
+        end
+      end
+
+      def reverse_dns_lookup(ip)
+        Timeout.timeout(config.dns_timeout) do
+          resolver.getname(ip)
+        end
+      rescue Resolv::ResolvError, Timeout::Error
+        nil
+      end
+
+      def forward_dns_lookup(hostname)
+        Timeout.timeout(config.dns_timeout) do
+          resolver.getaddresses(hostname)
+        end
+      rescue Resolv::ResolvError, Timeout::Error
+        []
+      end
+
+      def resolver
+        @resolver ||= Resolv::DNS.new
+      end
+
+      def config
+        BotVerification.configuration
+      end
+
+      def cache
+        config.cache
+      end
+
+      def report_verification_result(bot_type, verified, mode, ip, user_agent)
+        config.report_verification(
+          bot_type: bot_type,
+          verified: verified,
+          mode: mode,
+          ip: ip,
+          user_agent: user_agent
+        )
+        verified
+      end
+    end
+  end
+end

data/lib/bot_verification/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BotVerification
+  VERSION = "0.1.0"
+end

data/lib/bot_verification.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "rails"
+require "active_record"
+require "active_support"
+require "resolv"
+require "timeout"
+require "digest"
+require "net/http"
+require "openssl"
+require "json"
+
+require_relative "bot_verification/version"
+require_relative "bot_verification/configuration"
+require_relative "bot_verification/bot_patterns"
+require_relative "bot_verification/service"
+require_relative "bot_verification/ip_range_model"
+require_relative "bot_verification/ip_range_fetcher"
+require_relative "bot_verification/controller_concern"
+require_relative "bot_verification/refresh_job" if defined?(ActiveJob::Base)
+require_relative "bot_verification/railtie" if defined?(Rails::Railtie)
+
+module BotVerification
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+
+  class << self
+    attr_writer :configuration
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    # Convenience method to access the service
+    def verify(ip, user_agent, mode: :search_engines)
+      Service.verified_good_bot?(ip, user_agent, mode: mode)
+    end
+
+    # Check if an IP belongs to a known bot
+    def verify_ip(ip, bot_type)
+      Service.verified_bot_ip?(ip, bot_type)
+    end
+
+    # Detect bot type from user agent
+    def detect_bot(user_agent)
+      Service.detect_bot_type(user_agent)
+    end
+
+    # Refresh IP ranges from official sources
+    def refresh_ip_ranges!(bot_type = nil)
+      IpRangeFetcher.refresh!(bot_type)
+    end
+
+    # Get the IP range model class
+    def ip_range_model
+      configuration.ip_range_model_class
+    end
+
+    # Check if the IP ranges table exists
+    def table_exists?
+      return false unless ActiveRecord::Base.connected?
+
+      ActiveRecord::Base.connection.table_exists?(configuration.table_name)
+    end
+  end
+end

data/lib/generators/bot_verification/install_generator.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module BotVerification
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :table_name, type: :string, default: "bot_ip_ranges",
+        desc: "Name of the database table for storing bot IP ranges"
+
+      class_option :skip_migration, type: :boolean, default: false,
+        desc: "Skip creating the migration (if table already exists)"
+
+      class_option :skip_initializer, type: :boolean, default: false,
+        desc: "Skip creating the initializer"
+
+      class_option :skip_model, type: :boolean, default: false,
+        desc: "Skip creating the model file"
+
+      desc "Installs BotVerification gem: creates migration, initializer, and model"
+
+      def create_migration
+        return if options[:skip_migration]
+
+        if table_exists?
+          say "Table '#{table_name}' already exists. Skipping migration.", :yellow
+          return
+        end
+
+        migration_template "migration.rb.erb", "db/migrate/create_#{table_name}.rb"
+      end
+
+      def create_initializer
+        return if options[:skip_initializer]
+
+        template "initializer.rb.erb", "config/initializers/bot_verification.rb"
+      end
+
+      def create_model
+        return if options[:skip_model]
+
+        template "model.rb.erb", "app/models/bot_ip_range.rb"
+      end
+
+      def create_job
+        template "refresh_job.rb.erb", "app/jobs/refresh_bot_ip_ranges_job.rb"
+      end
+
+      def show_post_install_message
+        say ""
+        say "BotVerification installed successfully!", :green
+        say ""
+        say "Next steps:", :yellow
+        say "  1. Run migrations: rails db:migrate"
+        say "  2. Fetch initial IP ranges: rails bot_verification:refresh"
+        say "  3. Schedule daily refresh (add to Sidekiq-Cron or cron):"
+        say "     RefreshBotIpRangesJob.perform_later"
+        say ""
+        say "Usage in controllers:"
+        say "  include BotVerification::ControllerConcern"
+        say "  "
+        say "  if verified_good_bot?"
+        say "    # Request is from verified bot"
+        say "  end"
+        say ""
+      end
+
+      private
+
+      def table_name
+        options[:table_name]
+      end
+
+      def table_exists?
+        return false unless ActiveRecord::Base.connected?
+
+        ActiveRecord::Base.connection.table_exists?(table_name)
+      rescue StandardError
+        false
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/bot_verification/templates/initializer.rb.erb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# BotVerification configuration
+# See: https://github.com/mattkuklinski/bot_verification
+
+BotVerification.configure do |config|
+  # Table name for storing bot IP ranges
+  # Default: "bot_ip_ranges"
+  config.table_name = "<%= table_name %>"
+
+  # Skip DNS verification entirely (only use IP range matching)
+  # Set to true if DNS lookups add unacceptable latency for your project.
+  # When enabled, bots like Apple/Yandex/Baidu that don't publish IP ranges
+  # won't be verifiable - only Google/Bing/OpenAI/etc with published ranges.
+  # Default: false
+  # config.skip_dns_verification = false
+
+  # Timeout for each DNS lookup (reverse and forward) in seconds
+  # Only applies when skip_dns_verification is false
+  # Default: 1.0
+  # config.dns_timeout = 1.0
+
+  # Total timeout for all DNS operations in seconds
+  # Only applies when skip_dns_verification is false
+  # Default: 2.0
+  # config.dns_total_timeout = 2.0
+
+  # How long to cache verification results in Rails cache
+  # Default: 24.hours
+  # config.cache_ttl = 24.hours
+
+  # How long to cache results in session
+  # Default: 1.hour
+  # config.session_cache_ttl = 1.hour
+
+  # Cache key prefix for Rails cache
+  # Default: "bot_verification"
+  # config.cache_key_prefix = "bot_verification"
+
+  # Custom IP range model class name (optional)
+  # If you want to use your own model instead of the generated one
+  # config.ip_range_model_name = "BotIpRange"
+
+  # Error callback - integrate with your error tracking service
+  # Called when errors occur during IP range refresh
+  # config.on_error = ->(error, context) {
+  #   Airbrake.notify(error, context)
+  #   # or: Sentry.capture_exception(error, extra: context)
+  #   # or: Rails.logger.error("BotVerification error: #{error.message}")
+  # }
+
+  # Refresh complete callback - for notifications or monitoring
+  # Called after IP range refresh completes (success or partial failure)
+  # config.on_refresh_complete = ->(results) {
+  #   failures = results.select { |_, r| !r[:success] }
+  #   SlackNotifier.notify("Bot IPs refreshed. Failures: #{failures.keys}") if failures.any?
+  # }
+end

data/lib/generators/bot_verification/templates/migration.rb.erb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Create<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :<%= table_name %> do |t|
+      t.string :bot_type, null: false
+      t.string :cidr, null: false
+      t.integer :ip_version, null: false, default: 4
+      t.string :source_url
+      t.datetime :last_verified_at
+
+      t.timestamps
+    end
+
+    add_index :<%= table_name %>, :bot_type
+    add_index :<%= table_name %>, [:bot_type, :cidr], unique: true
+  end
+end

data/lib/generators/bot_verification/templates/model.rb.erb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Stores known IP ranges for legitimate search engine and AI bots
+# These ranges are refreshed daily by RefreshBotIpRangesJob
+#
+# @example Check if an IP belongs to a known bot
+#   BotIpRange.ip_belongs_to_bot?("66.249.66.1", :google) # => true
+#
+class BotIpRange < ApplicationRecord
+  include BotVerification::IpRangeModel
+
+  self.table_name = "<%= table_name %>"
+end