legitbot 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d775b4e4e434615989b8ef35aca9ac6498b06742ca6e8534f9bb4d6f3d5c9b04
-  data.tar.gz: 239069018d1b043ec99bce8496b3760fc189611f6aa83545835d5f71ce7f94c5
+  metadata.gz: dfc1b3322ff4f85957dabf6790d535f27f99feed47bdb4ff1bba65f6242d31a2
+  data.tar.gz: 32e842cc3d297b3afda0ef9b265121a11c363857a726819f886b441e6c53a53c
 SHA512:
-  metadata.gz: '08e08f94cbb4979566982d91a525f7d03618c9c10e85613bcebfd51f8f8f8b0dab669ba9cd1b67c7a6d6a36c23d23ad88b64996bf252bd884082045ae590d38c'
-  data.tar.gz: 97c4dd80bf62205ea4a03a0bc7570d57520611b4015f3bf9f66b450d78878a3fde1c9097e0ac79691e40d5f29e9ba676aaa3af70f71650adb61c7f4a0ee1ddf0
+  metadata.gz: 3654c256da13b37045425457a96ac9a8b41c5ae5c0cce49b7898170e2d23a66a5cb7e612503dc1d88dc2e1240dcb07c9ccc5d5aa8439f280144abe969dc0ae7b
+  data.tar.gz: 554e120d1001a71f455aedcd4d30397b22130279a6f99f8e022d40ade50427590dd52e982820399852ec52fbb2c96b7ea1461f82d8e88cb89acc053097136b32

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  Include:
+    - '**/Gemfile'
+    - '**/Rakefile'
+    - 'lib/**/*.rb'
+    - 'test/**/*.rb'
+  Exclude:
+    - 'pkg/**'

data/Gemfile

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 gemspec

data/Rakefile

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'bundler'
 require 'bump/tasks'
-require "rake/testtask"
+require 'rake/testtask'
 Bundler::GemHelper.install_tasks
 
 Bump.tag_by_default = true
 
 Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/*_test.rb']
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
   t.warning = true
   t.verbose = true
 end

data/legitbot.gemspec

@@ -17,9 +17,10 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.3.0'
   spec.add_dependency "irrc", ">= 0.2.1"
   spec.add_dependency "augmented_interval_tree", ">= 0.1.1"
-  spec.add_development_dependency "bump"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "bump", '>= 0.8.0'
+  spec.add_development_dependency "rake", '>= 12.3.0'
+  spec.add_development_dependency "rubocop", '>= 0.74.0'
+  spec.add_development_dependency "minitest", '>= 5.1.0'
 
   spec.files = `git ls-files`.split($/)
   spec.rdoc_options = ["--charset=UTF-8"]

data/lib/legitbot/ahrefs.rb

@@ -1,13 +1,18 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://ahrefs.com/robot
   class Ahrefs < BotMatch
-    Ranges = %w(54.36.148.0/24 54.36.149.0/24 54.36.150.0/24 195.154.122.0/24 195.154.123.0/24 195.154.126.0/24 195.154.127.0/24)
-
-    def valid?
-      ip = IPAddr.new @ip
-      Ranges.any? { |range| IPAddr.new(range).include? ip }
-    end
+    ip_ranges %w[
+      54.36.148.0/24
+      54.36.149.0/24
+      54.36.150.0/24
+      195.154.122.0/24
+      195.154.123.0/24
+      195.154.126.0/24
+      195.154.127.0/24
+    ]
   end
 
-  rule Legitbot::Ahrefs, %w(AhrefsBot)
+  rule Legitbot::Ahrefs, %w[AhrefsBot]
 end

data/lib/legitbot/apple.rb

@@ -1,20 +1,20 @@
+# frozen_string_literal: true
+
 require 'ipaddr'
 
-module Legitbot
+module Legitbot # :nodoc:
   # https://support.apple.com/en-us/HT204683
-
   class Apple < BotMatch
-    Range = IPAddr.new('17.0.0.0/8')
-
-    def valid?
-      ip = IPAddr.new @ip
-      Range.include? ip
-    end
+    ip_ranges '17.0.0.0/8'
   end
 
-  class Apple_as_Google < Apple
+  # https://support.apple.com/en-us/HT204683
+  # rubocop:disable Naming/ClassAndModuleCamelCase
+  class Apple_as_Google < BotMatch
+    ip_ranges '17.0.0.0/8'
   end
+  # rubocop:enable Naming/ClassAndModuleCamelCase
 
-  rule Legitbot::Apple, %w(Applebot)
-  rule Legitbot::Apple_as_Google, %w(Googlebot)
+  rule Legitbot::Apple, %w[Applebot]
+  rule Legitbot::Apple_as_Google, %w[Googlebot]
 end

data/lib/legitbot/baidu.rb

@@ -1,12 +1,10 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # http://help.baidu.com/question?prod_en=master&class=498&id=1000973
   class Baidu < BotMatch
-    ValidDomains = ["baidu.com.", "baidu.jp."]
-
-    def valid?
-      subdomain_of?(*Baidu::ValidDomains)
-    end
+    domains 'baidu.com.', 'baidu.jp.', reverse: false
   end
 
-  rule Legitbot::Baidu, %w(Baiduspider)
+  rule Legitbot::Baidu, %w[Baiduspider]
 end

data/lib/legitbot/bing.rb

@@ -1,12 +1,10 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/
   class Bing < BotMatch
-    ValidDomains = ["search.msn.com."]
-
-    def valid?
-      subdomain_of?(*Bing::ValidDomains) && reverse_resolves?
-    end
+    domains 'search.msn.com.'
   end
 
-  rule Legitbot::Bing, %w(Bingbot bingbot)
+  rule Legitbot::Bing, %w[Bingbot bingbot]
 end

data/lib/legitbot/botmatch.rb

@@ -1,5 +1,8 @@
-require 'resolv'
-require 'ipaddr'
+# frozen_string_literal: true
+
+require_relative 'config/resolver'
+require_relative 'validators/domains'
+require_relative 'validators/ip_ranges'
 
 module Legitbot
   ##
@@ -7,61 +10,31 @@ module Legitbot
   # +valid?+, +fake?+ and +detected_as+
   #
   class BotMatch
-    def initialize(ip, resolver_config = nil)
-      @dns = Resolv::DNS.new(resolver_config)
-      @ip = ip
-    end
-
-    ##
-    # Returns a Resolv::DNS::Name instance with
-    # the reverse name
-    def reverse_domain
-      @reverse_domain ||= @dns.getname(@ip)
-    rescue Resolv::ResolvError
-      @reverse_domain ||= nil
-    end
-
-    ##
-    # Returns a String with the reverse name
-    def reverse_name
-      reverse_domain&.to_s
-    end
-
-    ##
-    # Returns a String with IP created from the reverse name
-    def reversed_ip
-      return nil if reverse_name.nil?
+    include Legitbot::Validators::IpRanges
+    include Legitbot::Validators::Domains
 
-      @reverse_ip ||= @dns.getaddress(reverse_name)
-      @reverse_ip.to_s
-    end
-
-    def reverse_resolves?
-      @ip == reversed_ip
-    end
-
-    def subdomain_of?(*domains)
-      return false if reverse_name.nil?
-
-      domains.any? { |d|
-        reverse_domain.subdomain_of? Resolv::DNS::Name.create(d)
-      }
+    def initialize(ip)
+      @ip = ip
     end
 
     def detected_as
       self.class.name.split('::').last.downcase.to_sym
     end
 
+    def valid?
+      valid_ip? && valid_domain?
+    end
+
     def fake?
       !valid?
     end
 
-    def self.valid?(ip, resolver_config = nil)
-      self.new(ip, resolver_config).valid?
+    def self.valid?(ip)
+      new(ip).valid?
     end
 
-    def self.fake?(ip, resolver_config = nil)
-      self.new(ip, resolver_config).fake?
+    def self.fake?(ip)
+      new(ip).fake?
     end
   end
 end

data/lib/legitbot/config/resolver.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'resolv'
+
+module Legitbot
+  module Config
+    module Resolver # :nodoc:
+      def resolver_config(options = nil)
+        @resolver_config = options
+      end
+
+      def resolver
+        @resolver_config ||= Legitbot.resolver_config
+        @resolver ||= Resolv::DNS.new @resolver_config
+      end
+    end
+  end
+end

data/lib/legitbot/duckduckgo.rb

@@ -1,12 +1,20 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
-    ValidIPs = %w(50.16.241.113 50.16.241.114 50.16.241.117 50.16.247.234 52.204.97.54 52.5.190.19 54.197.234.188 54.208.100.253 23.21.227.69)
-
-    def valid?
-      DuckDuckGo::ValidIPs.include? @ip
-    end
+    ip_ranges %w[
+      50.16.241.113
+      50.16.241.114
+      50.16.241.117
+      50.16.247.234
+      52.204.97.54
+      52.5.190.19
+      54.197.234.188
+      54.208.100.253
+      23.21.227.69
+    ]
   end
 
-  rule Legitbot::DuckDuckGo, %w(DuckDuckBot)
+  rule Legitbot::DuckDuckGo, %w[DuckDuckBot]
 end

data/lib/legitbot/facebook.rb

@@ -1,48 +1,22 @@
-require 'ipaddr'
+# frozen_string_literal: true
+
 require 'irrc'
-require 'interval_tree'
 
-module Legitbot
+module Legitbot # :nodoc:
   # https://developers.facebook.com/docs/sharing/webmasters/crawler
-
   class Facebook < BotMatch
     AS = 'AS32934'
 
-    def valid?
-      ip = IPAddr.new(@ip)
-      Facebook.valid_ips[ip.ipv4? ? :ipv4 : :ipv6].search(ip.to_i).size > 0
-    end
-
-    @mutex = Mutex.new
-
-    def self.valid_ips
-      @mutex.synchronize { @ips ||= load_ips }
-    end
-
-    def self.reload!
-      @mutex.synchronize { @ips = load_ips }
-    end
-
-    def self.load_ips
-      whois.map do |(family, records)|
-        ranges = records.map do |cidr|
-          range = IPAddr.new(cidr).to_range
-          (range.begin.to_i..range.end.to_i)
-        end
-        [family, IntervalTree::Tree.new(ranges)]
-      end.to_h
-    end
-
-    def self.whois
+    ip_ranges do
       client = Irrc::Client.new
       client.query :radb, AS
       results = client.perform
 
-      %i(ipv4 ipv6).map do |family|
-        [family, results[AS][family][AS]]
-      end.to_h
+      %i[ipv4 ipv6].map do |family|
+        results[AS][family][AS]
+      end.flatten
     end
   end
 
-  rule Legitbot::Facebook, %w(Facebot facebookexternalhit/1.1)
+  rule Legitbot::Facebook, %w[Facebot facebookexternalhit/1.1]
 end

data/lib/legitbot/google.rb

@@ -1,14 +1,11 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://support.google.com/webmasters/answer/1061943
   # https://support.google.com/webmasters/answer/80553
-
   class Google < BotMatch
-    ValidDomains = ["google.com.", "googlebot.com."]
-
-    def valid?
-      subdomain_of?(*Google::ValidDomains) && reverse_resolves?
-    end
+    domains 'google.com.', 'googlebot.com.'
   end
 
-  rule Legitbot::Google, %w(Googlebot Mediapartners-Google AdsBot-Google)
+  rule Legitbot::Google, %w[Googlebot Mediapartners-Google AdsBot-Google]
 end

data/lib/legitbot/legitbot.rb

@@ -1,6 +1,14 @@
+# frozen_string_literal: true
+
+##
+# Bot lookup based on user agent
 module Legitbot
   @rules = []
 
+  class << self
+    attr_accessor :resolver_config
+  end
+
   ##
   # Lookup a bot based on its signature from +User-Agent+ header.
   #
@@ -10,15 +18,12 @@ module Legitbot
   # otherwise.
   # :yields: a found bot
   #
-  def self.bot(userAgent, ip, resolver_config = nil)
-    bots =
-      @rules.select { |rule|
-        rule[:fragments].any? {|f| userAgent.index f}
-      }.map { |rule|
-        rule[:class].new(ip, resolver_config)
-      }
+  def self.bot(user_agent, ip)
+    bots = @rules
+           .select { |rule| rule[:fragments].any? { |f| user_agent.index f } }
+           .map { |rule| rule[:class].new(ip) }
 
-    selected = bots.select { |b| b.valid? }.first if bots.size > 1
+    selected = bots.select(&:valid?).first if bots.size > 1
     selected = bots.last if selected.nil?
 
     if selected && block_given?
@@ -29,6 +34,6 @@ module Legitbot
   end
 
   def self.rule(clazz, fragments)
-    @rules << {:class => clazz, :fragments => fragments}
+    @rules << { class: clazz, fragments: fragments }
   end
 end

data/lib/legitbot/pinterest.rb

@@ -1,13 +1,10 @@
-module Legitbot
-  # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
+# frozen_string_literal: true
 
+module Legitbot # :nodoc:
+  # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
   class Pinterest < BotMatch
-    ValidDomains = ["pinterest.com."]
-
-    def valid?
-      subdomain_of?(*Pinterest::ValidDomains) && reverse_resolves?
-    end
+    domains 'pinterest.com.'
   end
 
-  rule Legitbot::Pinterest, %w(Pinterestbot Pinterest/0.2)
+  rule Legitbot::Pinterest, %w[Pinterestbot Pinterest/0.2]
 end

data/lib/legitbot/validators/domains.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'resolv'
+require 'ipaddr'
+
+module Legitbot
+  module Validators
+    #
+    # In a bot matcher:
+    # `domains 'search.msn.com', ...`
+    # `domains 'googlebot.com', reverse: false`
+    #
+    # `reverse` is true by default.
+    module Domains
+      class << self
+        def included(base)
+          base.extend ClassMethods
+        end
+      end
+
+      def valid_domain?
+        self.class.valid_domain?(@ip)
+      end
+
+      module ClassMethods # :nodoc:
+        include Legitbot::Config::Resolver
+
+        def domains(*list, reverse: true)
+          @valid_domains = list.flatten.map { |d| Resolv::DNS::Name.create(d) }
+          @validate_reverse_record = reverse
+        end
+
+        def check_domains?
+          instance_variable_defined?(:@valid_domains)
+        end
+
+        def valid_domain?(ip)
+          return true unless check_domains?
+          return true if @valid_domains.empty?
+
+          domains = reverse_domains(ip)
+          return false if domains.empty?
+
+          record = find_subdomain_record(domains)
+          return false unless record
+          return true unless @validate_reverse_record
+
+          ip == reverse_ip(record)
+        end
+
+        def reverse_domains(ip)
+          resolver.getnames(ip)
+        rescue Resolv::ResolvError
+          nil
+        end
+
+        def find_subdomain_record(domains)
+          domains.find do |d|
+            @valid_domains.any? { |vd| d.subdomain_of?(vd) }
+          end
+        end
+
+        def reverse_ip(record)
+          return nil if record.nil?
+
+          resolver.getaddress(record.to_s).to_s
+        end
+      end
+    end
+  end
+end

data/lib/legitbot/validators/ip_ranges.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'interval_tree'
+
+module Legitbot
+  module Validators
+    #
+    # In a bot matcher:
+    # `ip_ranges ip, range, ip, ...`
+    # `ip_ranges do [ip, range, ...]; end`
+    module IpRanges
+      class << self
+        def included(base)
+          base.extend ClassMethods
+        end
+      end
+
+      def valid_ip?
+        self.class.valid_ip?(@ip)
+      end
+
+      module ClassMethods # :nodoc:
+        FAMILIES = %i[ipv4 ipv6].freeze
+        EMPTY_GENERATOR = proc { [] }
+
+        def ip_ranges(*ips, &block)
+          @ip_ranges = partition_ips(ips.flatten) unless ips.empty?
+          @ip_ranges_loader = block_given? ? block : EMPTY_GENERATOR
+          @ip_loader_mutex = Mutex.new
+        end
+
+        def check_ranges?
+          instance_variable_defined?(:@ip_ranges_loader)
+        end
+
+        def valid_ip?(ip)
+          return true unless check_ranges?
+          return true if valid_ips.empty?
+
+          obj = IPAddr.new(ip)
+          ranges = valid_ips[obj.ipv4? ? :ipv4 : :ipv6].search(obj.to_i)
+          !ranges.empty?
+        end
+
+        def valid_ips
+          @ip_loader_mutex.synchronize do
+            @ip_ranges ||= load_ips
+          end
+        end
+
+        def reload_ips
+          @ip_loader_mutex.synchronize do
+            @ip_ranges = load_ips
+          end
+        end
+
+        def load_ips
+          partition_ips(@ip_ranges_loader.call)
+        end
+
+        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def partition_ips(ips)
+          return [] if ips.empty?
+
+          ips
+            .map { |cidr| IPAddr.new(cidr) }
+            .partition(&:ipv4?)
+            .each_with_index
+            .map do |list, index|
+              ranges = list.map(&:to_range).map do |r|
+                (r.begin.to_i..r.end.to_i)
+              end
+              [FAMILIES[index], IntervalTree::Tree.new(ranges)]
+            end.to_h
+        end
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+      end
+    end
+  end
+end

data/lib/legitbot/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Legitbot
-  VERSION = '0.3.2'
+  VERSION = '0.4.0'
 end

data/lib/legitbot/yandex.rb

@@ -1,17 +1,33 @@
-module Legitbot
-  # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
+# frozen_string_literal: true
 
+module Legitbot # :nodoc:
+  # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
   class Yandex < BotMatch
-    ValidDomains = ["yandex.ru.", "yandex.net.", "yandex.com."]
-
-    def valid?
-      subdomain_of?(*Yandex::ValidDomains) && reverse_resolves?
-    end
+    domains 'yandex.ru.', 'yandex.net.', 'yandex.com.'
   end
 
-  rule Legitbot::Yandex, %w(YandexBot YandexAccessibilityBot YandexMobileBot
-    YandexDirectDyn YandexScreenshotBot YandexImages YandexVideo YandexVideoParser
-    YandexMedia YandexBlogs YandexFavicons YandexWebmaster YandexPagechecker
-    YandexImageResizer YaDirectFetcher YandexCalendar YandexSitelinks YandexMetrika
-    YandexNews YandexVertis YandexSearchShop YandexVerticals)
+  rule Legitbot::Yandex, %w[
+    YandexBot
+    YandexAccessibilityBot
+    YandexMobileBot
+    YandexDirectDyn
+    YandexScreenshotBot
+    YandexImages
+    YandexVideo
+    YandexVideoParser
+    YandexMedia
+    YandexBlogs
+    YandexFavicons
+    YandexWebmaster
+    YandexPagechecker
+    YandexImageResizer
+    YaDirectFetcher
+    YandexCalendar
+    YandexSitelinks
+    YandexMetrika
+    YandexNews
+    YandexVertis
+    YandexSearchShop
+    YandexVerticals
+  ]
 end

data/lib/legitbot.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'legitbot/legitbot'
 require_relative 'legitbot/botmatch'