bot_verification 0.1.0

data/bot_verification.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/bot_verification/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "bot_verification"
+  spec.version = BotVerification::VERSION
+  spec.authors = ["Web Ventures Ltd"]
+  spec.email = ["gems@dev.webven.nz"]
+
+  spec.summary = "Verify legitimate search engine and AI bots by IP"
+  spec.description = "A Rails engine for verifying that requests claiming to be from " \
+                     "search engine bots (Google, Bing, etc.) and AI bots (GPTBot, PerplexityBot) " \
+                     "are actually from those services, using IP range matching and reverse DNS verification."
+  spec.homepage = "https://github.com/webventures/bot_verification"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|github|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", ">= 7.0"
+  spec.add_dependency "resolv", ">= 0.2"
+
+  spec.add_development_dependency "rspec", "~> 3.12"
+  spec.add_development_dependency "sqlite3", ">= 2.1"
+end

data/lib/bot_verification/bot_patterns.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module BotVerification
+  module BotPatterns
+    # =============================================================================
+    # SEARCH ENGINE BOTS
+    # =============================================================================
+
+    # User agent patterns for search engine bots
+    SEARCH_BOT_PATTERNS = {
+      google: /Googlebot|Google-Extended|Mediapartners-Google|AdsBot-Google|APIs-Google/i,
+      bing: /Bingbot|msnbot|BingPreview/i,
+      apple: /Applebot/i,
+      yandex: /YandexBot/i,
+      baidu: /Baiduspider/i
+    }.freeze
+
+    # Valid reverse DNS suffixes for search engine bots
+    SEARCH_BOT_DNS_SUFFIXES = {
+      google: %w[.googlebot.com .google.com .googleusercontent.com],
+      bing: %w[.search.msn.com],
+      apple: %w[.applebot.apple.com],
+      yandex: %w[.yandex.ru .yandex.net .yandex.com],
+      baidu: %w[.crawl.baidu.com .crawl.baidu.jp]
+    }.freeze
+
+    # Search engine bots with IP ranges in database
+    SEARCH_BOTS_WITH_IP_RANGES = %i[google bing].freeze
+
+    # Search engine bots that support reverse DNS verification
+    SEARCH_BOTS_WITH_DNS = %i[google bing apple yandex baidu].freeze
+
+    # =============================================================================
+    # AI BOTS
+    # =============================================================================
+
+    # User agent patterns for AI bots
+    AI_BOT_PATTERNS = {
+      openai_gptbot: /GPTBot/i,
+      openai_chatgpt: /ChatGPT-User/i,
+      openai_searchbot: /OAI-SearchBot/i,
+      anthropic: /ClaudeBot|Claude-Web|anthropic-ai/i,
+      perplexity: /PerplexityBot|Perplexity-User/i,
+      amazon: /Amazonbot/i,
+      cohere: /cohere-ai/i,
+      meta: /meta-externalagent/i,
+      bytedance: /Bytespider/i
+    }.freeze
+
+    # AI bots with officially published IP ranges (can be verified)
+    AI_BOTS_WITH_IP_RANGES = %i[openai_gptbot openai_chatgpt openai_searchbot perplexity amazon].freeze
+
+    # AI bots without official IP ranges (cannot be reliably verified)
+    AI_BOTS_WITHOUT_VERIFICATION = %i[anthropic cohere meta bytedance].freeze
+
+    # =============================================================================
+    # SOCIAL/OTHER BOTS (no verification available)
+    # =============================================================================
+
+    SOCIAL_BOT_PATTERNS = {
+      duckduckgo: /DuckDuckBot/i,
+      facebook: /facebookexternalhit|Facebot/i,
+      twitter: /Twitterbot/i,
+      linkedin: /LinkedInBot/i,
+      slack: /Slackbot/i,
+      discord: /Discordbot/i,
+      telegram: /TelegramBot/i
+    }.freeze
+
+    # =============================================================================
+    # COMBINED
+    # =============================================================================
+
+    ALL_BOT_PATTERNS = SEARCH_BOT_PATTERNS.merge(AI_BOT_PATTERNS).merge(SOCIAL_BOT_PATTERNS).freeze
+    ALL_BOTS_WITH_IP_RANGES = (SEARCH_BOTS_WITH_IP_RANGES + AI_BOTS_WITH_IP_RANGES).freeze
+
+    # Search engine bot types (as strings for database)
+    SEARCH_BOT_TYPES = %w[google bing].freeze
+
+    # AI bot types with IP ranges (as strings for database)
+    AI_BOT_TYPES = %w[openai_gptbot openai_chatgpt openai_searchbot perplexity amazon].freeze
+
+    # All bot types that have IP ranges
+    VALID_BOT_TYPES = (SEARCH_BOT_TYPES + AI_BOT_TYPES).freeze
+
+    # =============================================================================
+    # IP RANGE SOURCES
+    # =============================================================================
+
+    SEARCH_ENGINE_SOURCES = {
+      google: [
+        { url: "https://developers.google.com/static/search/apis/ipranges/googlebot.json", name: "Googlebot" },
+        { url: "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json", name: "Google Special" }
+      ],
+      bing: [
+        { url: "https://www.bing.com/toolbox/bingbot.json", name: "Bingbot" }
+      ]
+    }.freeze
+
+    AI_BOT_SOURCES = {
+      openai_gptbot: [
+        { url: "https://openai.com/gptbot.json", name: "GPTBot" }
+      ],
+      openai_chatgpt: [
+        { url: "https://openai.com/chatgpt-user.json", name: "ChatGPT-User" }
+      ],
+      openai_searchbot: [
+        { url: "https://openai.com/searchbot.json", name: "OAI-SearchBot" }
+      ],
+      perplexity: [
+        { url: "https://www.perplexity.ai/perplexitybot.json", name: "PerplexityBot" },
+        { url: "https://www.perplexity.ai/perplexity-user.json", name: "Perplexity-User" }
+      ],
+      amazon: [
+        { url: "https://developer.amazon.com/amazonbot/ip-addresses/", name: "Amazonbot" }
+      ]
+    }.freeze
+
+    IP_RANGE_SOURCES = SEARCH_ENGINE_SOURCES.merge(AI_BOT_SOURCES).freeze
+  end
+end

data/lib/bot_verification/configuration.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module BotVerification
+  class Configuration
+    # Table name for storing bot IP ranges
+    # Default: "bot_ip_ranges"
+    attr_accessor :table_name
+
+    # Skip DNS verification entirely (only use IP range matching)
+    # Set to true if DNS lookups are unacceptable for your project
+    # Default: false
+    attr_accessor :skip_dns_verification
+
+    # Verify SSL certificates when fetching IP ranges
+    # Set to false in development if you have certificate issues
+    # Default: true (always verify in production!)
+    attr_accessor :verify_ssl
+
+    # Timeout for each DNS lookup (reverse and forward) in seconds
+    # Default: 1.0
+    attr_accessor :dns_timeout
+
+    # Total timeout for all DNS operations in seconds
+    # Default: 2.0
+    attr_accessor :dns_total_timeout
+
+    # How long to cache verification results in Rails cache
+    # Default: 24 hours
+    attr_accessor :cache_ttl
+
+    # How long to cache results in session
+    # Default: 1 hour
+    attr_accessor :session_cache_ttl
+
+    # Cache key prefix for Rails cache
+    # Default: "bot_verification"
+    attr_accessor :cache_key_prefix
+
+    # Logger instance (defaults to Rails.logger)
+    attr_accessor :logger
+
+    # Custom IP range model class name (optional)
+    # If set, uses your own model instead of the built-in one
+    # The model must include BotVerification::IpRangeModel
+    attr_accessor :ip_range_model_name
+
+    # Error callback - called when errors occur during IP range refresh
+    # Useful for integrating with error tracking services (Airbrake, Sentry, etc.)
+    # @example
+    #   config.on_error = ->(error, context) { Airbrake.notify(error, context) }
+    attr_accessor :on_error
+
+    # Refresh complete callback - called after IP range refresh completes
+    # Useful for notifications or monitoring
+    # @example
+    #   config.on_refresh_complete = ->(results) { SlackNotifier.notify("Bot IPs refreshed: #{results}") }
+    attr_accessor :on_refresh_complete
+
+    # Verification callback - called after each bot verification attempt
+    # Useful for collecting statistics about bot traffic
+    # @example
+    #   config.on_verification = ->(result) {
+    #     # result = { bot_type: :google, verified: true, mode: :search_engines, ip: "...", user_agent: "..." }
+    #     BotRequestStats.increment(result[:bot_type], result[:verified])
+    #   }
+    attr_accessor :on_verification
+
+    def initialize
+      @table_name = "bot_ip_ranges"
+      @skip_dns_verification = false
+      @verify_ssl = true
+      @dns_timeout = 1.0
+      @dns_total_timeout = 2.0
+      @cache_ttl = 24.hours
+      @session_cache_ttl = 1.hour
+      @cache_key_prefix = "bot_verification"
+      @logger = nil
+      @ip_range_model_name = nil
+      @on_error = nil
+      @on_refresh_complete = nil
+      @on_verification = nil
+    end
+
+    # Report an error through the configured callback
+    def report_error(error, context = {})
+      return unless on_error.respond_to?(:call)
+
+      on_error.call(error, context)
+    rescue StandardError => e
+      logger.error("[BotVerification] Error in on_error callback: #{e.message}")
+    end
+
+    # Report refresh completion through the configured callback
+    def report_refresh_complete(results)
+      return unless on_refresh_complete.respond_to?(:call)
+
+      on_refresh_complete.call(results)
+    rescue StandardError => e
+      logger.error("[BotVerification] Error in on_refresh_complete callback: #{e.message}")
+    end
+
+    # Report verification attempt through the configured callback
+    def report_verification(result)
+      return unless on_verification.respond_to?(:call)
+
+      on_verification.call(result)
+    rescue StandardError => e
+      logger.error("[BotVerification] Error in on_verification callback: #{e.message}")
+    end
+
+    def logger
+      @logger || (defined?(Rails) ? Rails.logger : Logger.new($stdout))
+    end
+
+    def cache
+      @cache ||= if defined?(Rails) && Rails.respond_to?(:cache) && Rails.cache
+        Rails.cache
+      else
+        ActiveSupport::Cache::MemoryStore.new
+      end
+    end
+
+    # Get the IP range model class
+    def ip_range_model_class
+      if @ip_range_model_name
+        @ip_range_model_name.constantize
+      else
+        BotVerification::IpRange
+      end
+    end
+
+    # Validate configuration
+    def validate!
+      raise ConfigurationError, "table_name cannot be blank" if table_name.blank?
+      raise ConfigurationError, "dns_timeout must be positive" unless dns_timeout.positive?
+      raise ConfigurationError, "dns_total_timeout must be positive" unless dns_total_timeout.positive?
+    end
+  end
+end

data/lib/bot_verification/controller_concern.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module BotVerification
+  # Controller concern for bot verification with session-based caching
+  #
+  # @example Usage in a controller
+  #   class MyController < ApplicationController
+  #     include BotVerification::ControllerConcern
+  #
+  #     def show
+  #       if verified_good_bot?
+  #         # Serve full content to verified bots
+  #       else
+  #         # Rate limit or require authentication
+  #       end
+  #     end
+  #   end
+  #
+  module ControllerConcern
+    extend ActiveSupport::Concern
+
+    # Check if current request is from a verified good bot
+    # Uses session caching to avoid repeated verification calls
+    #
+    # @param mode [Symbol] Verification mode (see Service)
+    # @return [Boolean]
+    def verified_good_bot?(mode: :search_engines)
+      return @_verified_good_bot if defined?(@_verified_good_bot) && @_verification_mode == mode
+
+      @_verification_mode = mode
+      @_verified_good_bot = check_bot_with_session_cache(mode: mode)
+    end
+
+    # Check specifically if request is from a verified AI bot
+    #
+    # @param strict [Boolean] See Service#verified_ai_bot?
+    # @return [Boolean]
+    def verified_ai_bot?(strict: true)
+      cache_key = bot_session_cache_key
+      return false unless cache_key
+
+      cached = read_bot_session_cache(cache_key)
+      return true if cached && cached[:ai_verified] == true
+
+      result = Service.verified_ai_bot?(
+        request.remote_ip,
+        request.user_agent,
+        strict: strict
+      )
+
+      if result
+        write_bot_session_cache(cache_key, ai_verified: true, bot_type: detected_bot_type)
+      end
+
+      result
+    end
+
+    # Get the detected bot type for the current request (if any)
+    #
+    # @return [Symbol, nil]
+    def detected_bot_type
+      @_detected_bot_type ||= Service.detect_bot_type(request.user_agent)
+    end
+
+    # Check if user agent looks like a bot (regardless of verification)
+    #
+    # @return [Boolean]
+    def bot_user_agent?
+      detected_bot_type.present?
+    end
+
+    # Check if user agent looks like an AI bot
+    #
+    # @return [Boolean]
+    def ai_bot_user_agent?
+      Service.ai_bot_user_agent?(request.user_agent)
+    end
+
+    # Check if user agent looks like a search engine bot
+    #
+    # @return [Boolean]
+    def search_bot_user_agent?
+      Service.search_bot_user_agent?(request.user_agent)
+    end
+
+    private
+
+    def check_bot_with_session_cache(mode:)
+      cache_key = bot_session_cache_key
+      return false unless cache_key
+
+      cached = read_bot_session_cache(cache_key)
+      return cached[:verified] == true if cached
+
+      result = Service.verified_good_bot?(
+        request.remote_ip,
+        request.user_agent,
+        mode: mode
+      )
+
+      write_bot_session_cache(cache_key, verified: result, bot_type: detected_bot_type)
+      result
+    end
+
+    def bot_session_cache_key
+      return nil if request.remote_ip.blank? || request.user_agent.blank?
+
+      Service.session_cache_key(request.remote_ip, request.user_agent)
+    end
+
+    def read_bot_session_cache(key)
+      return nil unless bot_session_available?
+
+      cache_store = session[:bot_verification]
+      return nil unless cache_store.is_a?(Hash)
+
+      entry = cache_store[key]
+      return nil unless entry.is_a?(Hash)
+
+      cached_at = entry[:at]
+      ttl = BotVerification.configuration.session_cache_ttl.to_i
+      if cached_at && (Time.current.to_i - cached_at) > ttl
+        cache_store.delete(key)
+        return nil
+      end
+
+      entry
+    end
+
+    def write_bot_session_cache(key, data)
+      return unless bot_session_available?
+
+      session[:bot_verification] ||= {}
+
+      # Limit cache size to prevent session bloat
+      if session[:bot_verification].size >= 10
+        oldest_key = session[:bot_verification].min_by { |_, v| v[:at] || 0 }&.first
+        session[:bot_verification].delete(oldest_key) if oldest_key
+      end
+
+      session[:bot_verification][key] = data.merge(at: Time.current.to_i)
+    end
+
+    def bot_session_available?
+      defined?(session) && session.respond_to?(:[])
+    rescue ActionController::InvalidAuthenticityToken
+      false
+    end
+  end
+end

data/lib/bot_verification/ip_range_fetcher.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module BotVerification
+  # Fetches IP ranges from official sources
+  # Can be called manually or scheduled via cron/Sidekiq
+  #
+  # @example Run manually
+  #   BotVerification::IpRangeFetcher.refresh!
+  #
+  # @example Refresh specific bot type
+  #   BotVerification::IpRangeFetcher.refresh!(:google)
+  #
+  class IpRangeFetcher
+    class << self
+      # Refresh IP ranges from official sources
+      #
+      # @param bot_type [String, Symbol, nil] Specific bot type, or nil for all
+      # @return [Hash] Results for each bot type
+      def refresh!(bot_type = nil)
+        if bot_type
+          { bot_type.to_sym => refresh_bot_type(bot_type.to_s) }
+        else
+          refresh_all
+        end
+      end
+
+      private
+
+      def refresh_all
+        log_info("Starting refresh of all bot IP ranges")
+
+        results = {}
+        BotPatterns::IP_RANGE_SOURCES.each_key do |type|
+          results[type] = refresh_bot_type(type.to_s)
+        end
+
+        log_info("Completed refresh: #{results.inspect}")
+        config.report_refresh_complete(results)
+        results
+      end
+
+      def refresh_bot_type(bot_type)
+        sources = BotPatterns::IP_RANGE_SOURCES[bot_type.to_sym]
+        return { error: "Unknown bot type: #{bot_type}" } unless sources
+
+        ranges = []
+
+        sources.each do |source|
+          log_info("Fetching #{source[:name]} from #{source[:url]}")
+
+          begin
+            fetched = fetch_ranges_from_url(source[:url])
+            ranges.concat(fetched.map { |r| r.merge(source_url: source[:url]) })
+            log_info("Fetched #{fetched.size} ranges from #{source[:name]}")
+          rescue StandardError => e
+            log_error("Failed to fetch #{source[:name]}: #{e.message}")
+            config.report_error(e, bot_type: bot_type, source: source[:name], url: source[:url])
+          end
+        end
+
+        if ranges.any?
+          count = BotVerification.ip_range_model.import_ranges(bot_type, ranges)
+          log_info("Imported #{count} ranges for #{bot_type}")
+          { success: true, count: count }
+        else
+          log_warn("No ranges fetched for #{bot_type}")
+          { success: false, error: "No ranges fetched" }
+        end
+      end
+
+      def fetch_ranges_from_url(url)
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.verify_mode = config.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        http.open_timeout = 10
+        http.read_timeout = 30
+
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request["User-Agent"] = "BotVerification Gem/#{VERSION}"
+
+        response = http.request(request)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise "HTTP #{response.code}: #{response.message}"
+        end
+
+        parse_ip_ranges(response.body)
+      end
+
+      def parse_ip_ranges(body)
+        # Try direct JSON parse first
+        data = JSON.parse(body)
+        extract_prefixes(data)
+      rescue JSON::ParserError
+        # If direct parse fails, try to extract JSON from HTML (Amazon's format)
+        extract_json_from_html(body)
+      end
+
+      def extract_prefixes(data)
+        prefixes = data["prefixes"] || []
+
+        prefixes.filter_map do |prefix|
+          cidr = prefix["ipv4Prefix"] || prefix["ipv6Prefix"]
+          next unless cidr
+
+          # Ensure CIDR notation (Amazon uses bare IPs without /32)
+          cidr = "#{cidr}/32" if prefix["ipv4Prefix"] && !cidr.include?("/")
+          cidr = "#{cidr}/128" if prefix["ipv6Prefix"] && !cidr.include?("/")
+
+          ip_version = prefix["ipv4Prefix"] ? 4 : 6
+          { cidr: cidr, ip_version: ip_version }
+        end
+      end
+
+      def extract_json_from_html(html)
+        # Amazon embeds JSON in HTML with escaped quotes
+        # Look for JSON-like structure with prefixes
+        return [] unless html.include?("prefixes")
+
+        # Unescape HTML entities
+        unescaped = html.gsub("&quot;", '"').gsub("&amp;", "&").gsub("&lt;", "<").gsub("&gt;", ">")
+
+        # Try to find and parse JSON object containing prefixes
+        match = unescaped.match(/\{\s*"creationTime"[^{]*"prefixes"\s*:\s*\[.*?\]\s*\}/m)
+        return [] unless match
+
+        data = JSON.parse(match[0])
+        extract_prefixes(data)
+      rescue JSON::ParserError => e
+        log_error("Failed to parse embedded JSON: #{e.message}")
+        []
+      rescue StandardError => e
+        log_error("Failed to extract JSON from HTML: #{e.message}")
+        []
+      end
+
+      def config
+        BotVerification.configuration
+      end
+
+      def log_info(message)
+        config.logger.info("[BotVerification] #{message}")
+      end
+
+      def log_warn(message)
+        config.logger.warn("[BotVerification] #{message}")
+      end
+
+      def log_error(message)
+        config.logger.error("[BotVerification] #{message}")
+      end
+    end
+  end
+end