bot_verification 0.1.0

data/lib/generators/bot_verification/templates/refresh_job.rb.erb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Job to refresh bot IP ranges from official sources
+# Should be scheduled to run daily via cron/Sidekiq-Cron
+#
+# @example Run manually
+#   RefreshBotIpRangesJob.perform_now
+#
+# @example Run for specific bot type
+#   RefreshBotIpRangesJob.perform_now("google")
+#
+# @example Schedule with Sidekiq-Cron (in sidekiq.yml or initializer)
+#   RefreshBotIpRangesJob.perform_later
+#
+class RefreshBotIpRangesJob < ApplicationJob
+  queue_as :low
+
+  def perform(bot_type = nil)
+    BotVerification.refresh_ip_ranges!(bot_type)
+  end
+end

data/lib/tasks/bot_verification.rake

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+namespace :bot_verification do
+  desc "Refresh all bot IP ranges from official sources"
+  task refresh: :environment do
+    puts "Refreshing all bot IP ranges..."
+    results = BotVerification.refresh_ip_ranges!
+
+    results.each do |bot_type, result|
+      if result[:success]
+        puts "  #{bot_type}: #{result[:count]} ranges imported"
+      else
+        puts "  #{bot_type}: FAILED - #{result[:error]}"
+      end
+    end
+
+    puts "Done!"
+  end
+
+  desc "Refresh IP ranges for a specific bot type (e.g., google, bing, openai_gptbot)"
+  task :refresh_bot, [ :bot_type ] => :environment do |_t, args|
+    bot_type = args[:bot_type]
+    unless bot_type
+      puts "Usage: rake bot_verification:refresh_bot[google]"
+      exit 1
+    end
+
+    puts "Refreshing IP ranges for #{bot_type}..."
+    results = BotVerification.refresh_ip_ranges!(bot_type)
+
+    result = results[bot_type.to_sym]
+    if result[:success]
+      puts "  #{result[:count]} ranges imported"
+    else
+      puts "  FAILED - #{result[:error]}"
+    end
+
+    puts "Done!"
+  end
+
+  desc "Show statistics about stored bot IP ranges"
+  task stats: :environment do
+    puts "Bot IP Range Statistics:"
+    puts "-" * 50
+
+    stats = BotVerification.ip_range_model.stats
+
+    stats.each do |bot_type, data|
+      last_updated = data[:last_updated]&.strftime("%Y-%m-%d %H:%M") || "Never"
+      puts "  #{bot_type.ljust(20)} #{data[:count].to_s.rjust(5)} ranges  (Last: #{last_updated})"
+    end
+
+    total = stats.values.sum { |d| d[:count] }
+    puts "-" * 50
+    puts "  #{'Total'.ljust(20)} #{total.to_s.rjust(5)} ranges"
+  end
+
+  desc "Clear all bot IP range caches"
+  task clear_cache: :environment do
+    puts "Clearing all bot IP range caches..."
+    BotVerification.ip_range_model.clear_all_caches
+    puts "Done!"
+  end
+
+  desc "Verify if an IP belongs to a known bot (e.g., rake bot_verification:verify_ip[66.249.66.1,google])"
+  task :verify_ip, [ :ip, :bot_type ] => :environment do |_t, args|
+    ip = args[:ip]
+    bot_type = args[:bot_type]
+
+    unless ip && bot_type
+      puts "Usage: rake bot_verification:verify_ip[66.249.66.1,google]"
+      exit 1
+    end
+
+    puts "Verifying IP #{ip} for bot type #{bot_type}..."
+
+    # Check IP range first
+    in_range = BotVerification.ip_range_model.ip_belongs_to_bot?(ip, bot_type)
+    puts "  IP in known ranges: #{in_range}"
+
+    # Try full verification
+    verified = BotVerification.verify_ip(ip, bot_type.to_sym)
+    puts "  Full verification: #{verified}"
+  end
+
+  desc "Check if table exists"
+  task check_table: :environment do
+    if BotVerification.table_exists?
+      puts "Table '#{BotVerification.configuration.table_name}' exists."
+    else
+      puts "Table '#{BotVerification.configuration.table_name}' does NOT exist."
+      puts "Run: rails generate bot_verification:install"
+    end
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: bot_verification
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Web Ventures Ltd
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: resolv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: A Rails engine for verifying that requests claiming to be from search
+  engine bots (Google, Bing, etc.) and AI bots (GPTBot, PerplexityBot) are actually
+  from those services, using IP range matching and reverse DNS verification.
+email:
+- gems@dev.webven.nz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bot_verification.gemspec
+- lib/bot_verification.rb
+- lib/bot_verification/bot_patterns.rb
+- lib/bot_verification/configuration.rb
+- lib/bot_verification/controller_concern.rb
+- lib/bot_verification/ip_range_fetcher.rb
+- lib/bot_verification/ip_range_model.rb
+- lib/bot_verification/railtie.rb
+- lib/bot_verification/refresh_job.rb
+- lib/bot_verification/service.rb
+- lib/bot_verification/version.rb
+- lib/generators/bot_verification/install_generator.rb
+- lib/generators/bot_verification/templates/initializer.rb.erb
+- lib/generators/bot_verification/templates/migration.rb.erb
+- lib/generators/bot_verification/templates/model.rb.erb
+- lib/generators/bot_verification/templates/refresh_job.rb.erb
+- lib/tasks/bot_verification.rake
+homepage: https://github.com/webventures/bot_verification
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/webventures/bot_verification
+  source_code_uri: https://github.com/webventures/bot_verification
+  changelog_uri: https://github.com/webventures/bot_verification/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.23
+signing_key:
+specification_version: 4
+summary: Verify legitimate search engine and AI bots by IP
+test_files: []