legitbot 0.3.2 → 1.0.0

data/lib/legitbot/google.rb

@@ -1,14 +1,11 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://support.google.com/webmasters/answer/1061943
   # https://support.google.com/webmasters/answer/80553
-
   class Google < BotMatch
-    ValidDomains = ["google.com.", "googlebot.com."]
-
-    def valid?
-      subdomain_of?(*Google::ValidDomains) && reverse_resolves?
-    end
+    domains 'google.com.', 'googlebot.com.'
   end
 
-  rule Legitbot::Google, %w(Googlebot Mediapartners-Google AdsBot-Google)
+  rule Legitbot::Google, %w[Googlebot Mediapartners-Google AdsBot-Google]
 end

data/lib/legitbot/legitbot.rb

@@ -1,6 +1,14 @@
+# frozen_string_literal: true
+
+##
+# Bot lookup based on user agent
 module Legitbot
   @rules = []
 
+  class << self
+    attr_accessor :resolver_config
+  end
+
   ##
   # Lookup a bot based on its signature from +User-Agent+ header.
   #
@@ -10,15 +18,12 @@ module Legitbot
   # otherwise.
   # :yields: a found bot
   #
-  def self.bot(userAgent, ip, resolver_config = nil)
-    bots =
-      @rules.select { |rule|
-        rule[:fragments].any? {|f| userAgent.index f}
-      }.map { |rule|
-        rule[:class].new(ip, resolver_config)
-      }
+  def self.bot(user_agent, ip)
+    bots = @rules
+           .select { |rule| rule[:fragments].any? { |f| user_agent.index f } }
+           .map { |rule| rule[:class].new(ip) }
 
-    selected = bots.select { |b| b.valid? }.first if bots.size > 1
+    selected = bots.select(&:valid?).first if bots.size > 1
     selected = bots.last if selected.nil?
 
     if selected && block_given?
@@ -29,6 +34,6 @@ module Legitbot
   end
 
   def self.rule(clazz, fragments)
-    @rules << {:class => clazz, :fragments => fragments}
+    @rules << { class: clazz, fragments: fragments }
   end
 end

data/lib/legitbot/oracle.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
+  class Oracle < BotMatch
+    ip_ranges '148.64.56.0/24'
+  end
+
+  rule Legitbot::Oracle, %w[GrapeshotCrawler]
+end

data/lib/legitbot/pinterest.rb

@@ -1,13 +1,10 @@
-module Legitbot
-  # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
+# frozen_string_literal: true
 
+module Legitbot # :nodoc:
+  # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
   class Pinterest < BotMatch
-    ValidDomains = ["pinterest.com."]
-
-    def valid?
-      subdomain_of?(*Pinterest::ValidDomains) && reverse_resolves?
-    end
+    domains 'pinterest.com.'
   end
 
-  rule Legitbot::Pinterest, %w(Pinterestbot Pinterest/0.2)
+  rule Legitbot::Pinterest, %w[Pinterestbot Pinterest/0.2]
 end

data/lib/legitbot/twitter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started
+  # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards
+  class Twitter < BotMatch
+    ip_ranges %w[
+      199.16.156.0/22
+      199.59.148.0/22
+    ]
+  end
+
+  rule Legitbot::Twitter, %w[Twitterbot]
+end

data/lib/legitbot/validators/domains.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'resolv'
+require 'ipaddr'
+
+module Legitbot
+  module Validators
+    #
+    # In a bot matcher:
+    # `domains 'search.msn.com', ...`
+    # `domains 'googlebot.com', reverse: false`
+    #
+    # `reverse` is true by default.
+    module Domains
+      class << self
+        def included(base)
+          base.extend ClassMethods
+        end
+      end
+
+      def valid_domain?
+        self.class.valid_domain?(@ip)
+      end
+
+      module ClassMethods # :nodoc:
+        include Legitbot::Config::Resolver
+
+        def domains(*list, reverse: true)
+          @valid_domains = list.flatten.map { |d| Resolv::DNS::Name.create(d) }
+          @validate_reverse_record = reverse
+        end
+
+        def check_domains?
+          instance_variable_defined?(:@valid_domains)
+        end
+
+        def valid_domain?(ip)
+          return true unless check_domains?
+          return true if @valid_domains.empty?
+
+          domains = reverse_domains(ip)
+          return false if domains.empty?
+
+          record = find_subdomain_record(domains)
+          return false unless record
+          return true unless @validate_reverse_record
+
+          ip == reverse_ip(record)
+        end
+
+        def reverse_domains(ip)
+          resolver.getnames(ip)
+        rescue Resolv::ResolvError
+          nil
+        end
+
+        def find_subdomain_record(domains)
+          domains.find do |d|
+            @valid_domains.any? { |vd| d.subdomain_of?(vd) }
+          end
+        end
+
+        def reverse_ip(record)
+          return nil if record.nil?
+
+          resolver.getaddress(record.to_s).to_s
+        end
+      end
+    end
+  end
+end

data/lib/legitbot/validators/ip_ranges.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'interval_tree'
+
+module Legitbot
+  module Validators
+    #
+    # In a bot matcher:
+    # `ip_ranges ip, range, ip, ...`
+    # `ip_ranges do [ip, range, ...]; end`
+    module IpRanges
+      class << self
+        def included(base)
+          base.extend ClassMethods
+        end
+      end
+
+      def valid_ip?
+        self.class.valid_ip?(@ip)
+      end
+
+      module ClassMethods # :nodoc:
+        FAMILIES = %i[ipv4 ipv6].freeze
+        EMPTY_GENERATOR = proc { [] }
+
+        def ip_ranges(*ips, &block)
+          @ip_ranges = partition_ips(ips.flatten) unless ips.empty?
+          @ip_ranges_loader = block_given? ? block : EMPTY_GENERATOR
+          @ip_loader_mutex = Mutex.new
+        end
+
+        def check_ranges?
+          instance_variable_defined?(:@ip_ranges_loader)
+        end
+
+        def valid_ip?(ip)
+          return true unless check_ranges?
+          return true if valid_ips.empty?
+
+          obj = IPAddr.new(ip)
+          ranges = valid_ips[obj.ipv4? ? :ipv4 : :ipv6].search(obj.to_i)
+          !ranges.empty?
+        end
+
+        def valid_ips
+          @ip_loader_mutex.synchronize do
+            @ip_ranges ||= load_ips
+          end
+        end
+
+        def reload_ips
+          @ip_loader_mutex.synchronize do
+            @ip_ranges = load_ips
+          end
+        end
+
+        def load_ips
+          partition_ips(@ip_ranges_loader.call)
+        end
+
+        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def partition_ips(ips)
+          return [] if ips.empty?
+
+          ips
+            .map { |cidr| IPAddr.new(cidr) }
+            .partition(&:ipv4?)
+            .each_with_index
+            .map do |list, index|
+              ranges = list.map(&:to_range).map do |r|
+                (r.begin.to_i..r.end.to_i)
+              end
+              [FAMILIES[index], IntervalTree::Tree.new(ranges)]
+            end.to_h
+        end
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+      end
+    end
+  end
+end

data/lib/legitbot/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Legitbot
-  VERSION = '0.3.2'
+  VERSION = '1.0.0'
 end

data/lib/legitbot/yandex.rb

@@ -1,17 +1,46 @@
-module Legitbot
-  # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
+# frozen_string_literal: true
 
+module Legitbot # :nodoc:
+  # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
   class Yandex < BotMatch
-    ValidDomains = ["yandex.ru.", "yandex.net.", "yandex.com."]
-
-    def valid?
-      subdomain_of?(*Yandex::ValidDomains) && reverse_resolves?
-    end
+    domains 'yandex.ru.', 'yandex.net.', 'yandex.com.'
   end
 
-  rule Legitbot::Yandex, %w(YandexBot YandexAccessibilityBot YandexMobileBot
-    YandexDirectDyn YandexScreenshotBot YandexImages YandexVideo YandexVideoParser
-    YandexMedia YandexBlogs YandexFavicons YandexWebmaster YandexPagechecker
-    YandexImageResizer YaDirectFetcher YandexCalendar YandexSitelinks YandexMetrika
-    YandexNews YandexVertis YandexSearchShop YandexVerticals)
+  rule Legitbot::Yandex, %w[
+    YandexAccessibilityBot
+    YandexAdNet
+    YandexBlogs
+    YandexBot/
+    YandexCalendar
+    YandexDirect/
+    YandexDirectDyn
+    YandexFavicons
+    YaDirectFetcher
+    YandexForDomain
+    YandexImages
+    YandexImageResizer
+    YandexMobileBot
+    YandexMarket
+    YandexMedia
+    YandexMetrika
+    YandexMobileScreenShotBot
+    YandexNews
+    YandexOntoDB
+    YandexOntoDBAPI
+    YandexPagechecker
+    YandexPartner
+    YandexRCA
+    YandexSearchShop
+    YandexSitelinks
+    YandexSpravBot
+    YandexTracker
+    YandexTurbo
+    YandexVertis
+    YandexVerticals
+    YandexVideo
+    YandexVideoParser
+    YandexWebmaster
+    YandexScreenshotBot
+    YandexMedianaBot
+  ]
 end

data/test/ahrefs_test.rb

@@ -1,28 +1,36 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class AhrefsTest < Minitest::Test
   def test_malicious_ip
-    ip = "149.210.164.47"
+    ip = '149.210.164.47'
     match = Legitbot::Ahrefs.new ip
     assert !match.valid?, msg: "#{ip} is not a real Ahrefs IP"
   end
 
   def test_valid_ip
-    ip = "54.36.148.0"
+    ip = '54.36.148.0'
     match = Legitbot::Ahrefs.new ip
     assert match.valid?, msg: "#{ip} is a valid Ahrefs IP"
   end
 
   def test_malicious_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", "149.210.164.47")
-    assert bot, msg: "Ahrefs detected from User-Agent"
-    assert !bot.valid?, msg: "Not a valid Ahrefs"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Ahrefs detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Ahrefs'
   end
 
   def test_valid_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", "54.36.148.0")
-    assert bot, msg: "Ahrefs detected from User-Agent"
-    assert bot.valid?, msg: "Valid Ahrefs"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
+      '54.36.148.0'
+    )
+    assert bot, msg: 'Ahrefs detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Ahrefs'
   end
 end

data/test/apple_as_google_test.rb

@@ -1,22 +1,27 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class AppleAsGoogleTest < Minitest::Test
   def test_valid_ip
-    ip = "17.58.98.60"
+    ip = '17.58.98.60'
     match = Legitbot::Apple_as_Google.new(ip)
     assert match.valid?, msg: "#{ip} is a valid Applebot IP"
   end
 
   def test_invalid_ip
-    ip = "127.0.0.1"
+    ip = '127.0.0.1'
     match = Legitbot::Apple_as_Google.new(ip)
     assert match.fake?, msg: "#{ip} is a fake Applebot IP"
   end
 
   def test_user_agent
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "17.58.98.60")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+      '17.58.98.60'
+    )
     assert_equal :apple_as_google, bot.detected_as
-    assert bot.valid?, msg: "A valid Applebot User-agent and IP"
+    assert bot.valid?, msg: 'A valid Applebot User-agent and IP'
   end
 end

data/test/apple_test.rb

@@ -1,22 +1,29 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class AppleTest < Minitest::Test
   def test_valid_ip
-    ip = "17.58.98.60"
+    ip = '17.58.98.60'
     match = Legitbot::Apple.new(ip)
     assert match.valid?, msg: "#{ip} is a valid Applebot IP"
   end
 
   def test_invalid_ip
-    ip = "127.0.0.1"
+    ip = '127.0.0.1'
     match = Legitbot::Apple.new(ip)
     assert match.fake?, msg: "#{ip} is a fake Applebot IP"
   end
 
+  # rubocop:disable Layout/LineLength
   def test_user_agent
-    bot = Legitbot.bot("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", "17.58.98.60")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
+      '17.58.98.60'
+    )
     assert_equal :apple, bot.detected_as
-    assert bot.valid?, msg: "A valid Applebot User-agent and IP"
+    assert bot.valid?, msg: 'A valid Applebot User-agent and IP'
   end
+  # rubocop:enable Layout/LineLength
 end

data/test/botmatch_test.rb

@@ -1,29 +1,11 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class BotMatchTest < Minitest::Test
-  def test_reverse_name
-    match = Legitbot::BotMatch.new "66.249.64.141"
-    assert_equal "crawl-66-249-64-141.googlebot.com", match.reverse_name
-  end
-
-  def test_reverse_ip
-    match = Legitbot::BotMatch.new "66.249.64.141"
-    assert_equal "66.249.64.141", match.reversed_ip
-  end
-
-  def test_reverse_resolves
-    match = Legitbot::BotMatch.new "66.249.64.141"
-    assert_equal true, match.reverse_resolves?
-  end
-
-  def test_reverse_doesnt_resolve
-    match = Legitbot::BotMatch.new "5.140.70.64"
-    assert !match.reverse_resolves?
-  end
-
   def test_valid_class_syntax
-    assert Legitbot::Google.valid?("66.249.64.141"), msg: "Valid Googlebot"
-    assert Legitbot::Google.fake?("149.210.164.47"), msg: "Fake Googlebot"
+    assert Legitbot::Google.valid?('66.249.64.141'), msg: 'Valid Googlebot'
+    assert Legitbot::Google.fake?('149.210.164.47'), msg: 'Fake Googlebot'
   end
 end