legitbot 0.3.2 → 1.0.0

data/test/facebook_test.rb

@@ -1,20 +1,24 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 module Legitbot
   class Facebook
+    # rubocop:disable Layout/LineLength
     def self.whois
       {
-        ipv4: ["69.63.176.0/20", "66.220.144.0/20", "66.220.144.0/21", "69.63.184.0/21", "69.63.176.0/21", "74.119.76.0/22", "69.171.255.0/24", "173.252.64.0/18", "69.171.224.0/19", "69.171.224.0/20", "103.4.96.0/22", "69.63.176.0/24", "173.252.64.0/19", "173.252.70.0/24", "31.13.64.0/18", "31.13.24.0/21", "66.220.152.0/21", "66.220.159.0/24", "69.171.239.0/24", "69.171.240.0/20", "31.13.64.0/19", "31.13.64.0/24", "31.13.65.0/24", "31.13.67.0/24", "31.13.68.0/24", "31.13.69.0/24", "31.13.70.0/24", "31.13.71.0/24", "31.13.72.0/24", "31.13.73.0/24", "31.13.74.0/24", "31.13.75.0/24", "31.13.76.0/24", "31.13.77.0/24", "31.13.96.0/19", "31.13.66.0/24", "173.252.96.0/19", "69.63.178.0/24", "31.13.78.0/24", "31.13.79.0/24", "31.13.80.0/24", "31.13.82.0/24", "31.13.83.0/24", "31.13.84.0/24", "31.13.85.0/24", "31.13.86.0/24", "31.13.87.0/24", "31.13.88.0/24", "31.13.89.0/24", "31.13.90.0/24", "31.13.91.0/24", "31.13.92.0/24", "31.13.93.0/24", "31.13.94.0/24", "31.13.95.0/24", "69.171.253.0/24", "69.63.186.0/24", "31.13.81.0/24", "179.60.192.0/22", "179.60.192.0/24", "179.60.193.0/24", "179.60.194.0/24", "179.60.195.0/24", "185.60.216.0/22", "45.64.40.0/22", "185.60.216.0/24", "185.60.217.0/24", "185.60.218.0/24", "185.60.219.0/24", "129.134.0.0/16", "157.240.0.0/16", "157.240.8.0/24", "157.240.0.0/24", "157.240.1.0/24", "157.240.2.0/24", "157.240.3.0/24", "157.240.4.0/24", "157.240.5.0/24", "157.240.6.0/24", "157.240.7.0/24", "157.240.9.0/24", "157.240.10.0/24", "157.240.16.0/24", "157.240.19.0/24", "157.240.11.0/24", "157.240.12.0/24", "157.240.13.0/24", "157.240.14.0/24", "157.240.15.0/24", "157.240.17.0/24", "157.240.18.0/24", "157.240.20.0/24", "157.240.21.0/24", "157.240.22.0/24", "157.240.23.0/24", "157.240.0.0/17", "69.171.250.0/24", "157.240.24.0/24", "157.240.25.0/24", "199.201.64.0/24", "199.201.65.0/24", "199.201.64.0/22", "204.15.20.0/22", "157.240.192.0/24", "129.134.0.0/17", "157.240.198.0/24"],
+        ipv4: ['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24'],
         ipv6: []
       }
     end
+    # rubocop:enable Layout/LineLength
   end
 end
 
 class FacebookTest < Minitest::Test
   def test_valid_ip
-    ip = "69.63.186.89"
+    ip = '69.63.186.89'
     match = Legitbot::Facebook.new(ip)
     assert match.valid?, msg: "#{ip} is a valid Facebook IP"
 
@@ -24,25 +28,36 @@ class FacebookTest < Minitest::Test
   end
 
   def test_invalid_ip
-    ip = "127.0.0.1"
+    ip = '127.0.0.1'
     match = Legitbot::Facebook.new(ip)
     assert match.fake?, msg: "#{ip} is a fake Facebook IP"
   end
 
+  # rubocop:disable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
   def test_user_agent
-    Legitbot.bot("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "31.13.76.56") do |bot|
+    Legitbot.bot(
+      'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
+      '31.13.76.56'
+    ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?, msg: "true Facebook"
+      assert bot.valid?, msg: 'true Facebook'
     end
 
-    Legitbot.bot("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "173.252.87.8") do |bot|
+    Legitbot.bot(
+      'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
+      '173.252.87.8'
+    ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?, msg: "true Facebook"
+      assert bot.valid?, msg: 'true Facebook'
     end
 
-    Legitbot.bot("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0", "92.243.181.7") do |bot|
-      assert_equal :facebook, bot.detected_as
-      assert bot.fake?, msg: "fake Facebook"
+    Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
+      '92.243.181.7'
+    ) do |bot|
+      assert %i[facebook twitter].include?(bot.detected_as)
+      assert bot.fake?, msg: 'fake Facebook'
     end
   end
+  # rubocop:enable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
 end

data/test/google_test.rb

@@ -1,42 +1,52 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class GoogleTest < Minitest::Test
   def test_malicious_ip
-    ip = "149.210.164.47"
+    ip = '149.210.164.47'
     match = Legitbot::Google.new ip
-    reverse_name = match.reverse_name
-    assert !match.subdomain_of?("googlebot.com."), msg: "#{reverse_name} is not a subdomain of googlebot.com"
     assert !match.valid?, msg: "#{ip} is not a real Googlebot IP"
   end
 
   def test_valid_ip
-    ip = "66.249.64.141"
+    ip = '66.249.64.141'
     match = Legitbot::Google.new ip
-    reverse_name = match.reverse_name
-    assert match.subdomain_of?("googlebot.com."), msg: "#{reverse_name} is a subdomain of googlebot.com"
     assert match.valid?, msg: "#{ip} is a valid Googlebot IP"
   end
 
   def test_malicious_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "149.210.164.47")
-    assert bot, msg: "Googlebot detected from User-Agent"
-    assert !bot.valid?, msg: "Not a valid Googlebot"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Googlebot detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Googlebot'
   end
 
   def test_valid_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.64.141")
-    assert bot, msg: "Googlebot detected from User-Agent"
-    assert bot.valid?, msg: "Valid Googlebot"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+      '66.249.64.141'
+    )
+    assert bot, msg: 'Googlebot detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Googlebot'
   end
 
   def test_valid_name
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.64.141")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+      '66.249.64.141'
+    )
     assert_equal :google, bot.detected_as
   end
 
   def test_fake_name
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "81.1.172.108")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+      '81.1.172.108'
+    )
     assert_equal :google, bot.detected_as
   end
 end

data/test/legitbot/validators/domains_test.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+module Legitbot
+  module Validators
+    class NoDomains
+      include Domains
+    end
+
+    class DomainMatch
+      include Domains
+      domains 'search.msn.com', reverse: false
+
+      @resolver = Minitest::Mock.new
+      @resolver.expect(:getnames,
+                       ['po18-218.co2-6nf-srch-2b.ntwk.msn.net',
+                        'msnbot-157-55-39-132.search.msn.com'].map do |d|
+                         Resolv::DNS::Name.create(d)
+                       end, [String])
+      @resolver.expect(:getnames,
+                       ['crawl-66-249-64-141.googlebot.com'].map do |d|
+                         Resolv::DNS::Name.create(d)
+                       end, [String])
+    end
+
+    class ReverseMatch
+      include Domains
+      domains 'search.msn.com'
+    end
+
+    class DomainsTest < Minitest::Test
+      def test_no_domains
+        assert NoDomains.valid_domain?('127.0.0.1')
+      end
+
+      def test_reverse_domain
+        assert DomainMatch.valid_domain?('127.0.0.1')
+        refute DomainMatch.valid_domain?('127.0.0.1')
+      end
+
+      def test_reverse_ip
+        dlist = ['po18-218.co2-6nf-srch-2b.ntwk.msn.net',
+                 'msnbot-157-55-39-132.search.msn.com'].map do |d|
+          Resolv::DNS::Name.create(d)
+        end
+
+        ReverseMatch.resolver.stub :getnames, dlist do
+          ReverseMatch.resolver.stub :getaddress, '127.0.0.1' do
+            assert ReverseMatch.valid_domain?('127.0.0.1')
+            refute ReverseMatch.valid_domain?('127.0.0.2')
+          end
+        end
+      end
+    end
+  end
+end

data/test/legitbot/validators/ip_ranges_test.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+module Legitbot
+  module Validators
+    class NoRanges
+      include IpRanges
+    end
+
+    class ArrayRanges
+      include IpRanges
+      ip_ranges '66.220.144.0/21', '2a03:2880:f234::/48'
+    end
+
+    class FlattenRanges
+      include IpRanges
+      ip_ranges %w[66.220.144.0/21 2a03:2880:f234::/48]
+    end
+
+    class EmptyRanges
+      include IpRanges
+      ip_ranges
+
+      def initialize(ip)
+        @ip = ip
+      end
+    end
+
+    class LoadRanges
+      include IpRanges
+
+      @i = 0
+      ip_ranges do
+        @i += 1
+        [@i.odd? ? '127.0.0.0/8' : '192.168.0.0/16']
+      end
+
+      def self.counter
+        @i
+      end
+
+      def initialize(ip)
+        @ip = ip
+      end
+    end
+
+    class IpRangesTest < Minitest::Test
+      def test_partition_method
+        empty = NoRanges.partition_ips([])
+        assert_empty empty
+      end
+
+      def test_ipv6_partition
+        ipv6 = NoRanges.partition_ips(['2a03:2880:f234::/48'])
+        assert_nil ipv6[:ipv4].top_node
+        refute_nil ipv6[:ipv6].top_node
+      end
+
+      def test_ipv4_partition
+        ipv4 = NoRanges.partition_ips(['66.220.144.0/21'])
+        refute_nil ipv4[:ipv4].top_node
+        assert_nil ipv4[:ipv6].top_node
+      end
+
+      def test_no_ranges
+        assert NoRanges.valid_ip?('127.0.0.1')
+      end
+
+      def test_empty_matcher
+        assert_empty EmptyRanges.valid_ips
+        assert_empty EmptyRanges.load_ips
+        assert EmptyRanges.valid_ip?('127.0.0.0')
+        assert EmptyRanges.valid_ip?('66.220.144.1')
+        assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
+
+        matcher = EmptyRanges.new '127.0.0.0'
+        assert matcher.valid_ip?
+      end
+
+      def test_matcher_array
+        assert ArrayRanges.valid_ip?('66.220.144.1')
+        assert ArrayRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
+        refute ArrayRanges.valid_ip?('66.220.143.1')
+        refute ArrayRanges.valid_ip?('2a03:2880:f233:0:0:0:0:1')
+      end
+
+      def test_flatten
+        assert FlattenRanges.valid_ip?('66.220.144.1')
+        assert FlattenRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
+        refute FlattenRanges.valid_ip?('66.220.143.1')
+        refute FlattenRanges.valid_ip?('2a03:2880:f233:0:0:0:0:1')
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      def test_matcher_loader
+        assert_equal 0, LoadRanges.counter
+        assert LoadRanges.new('127.127.127.127').valid_ip?
+        refute LoadRanges.new('10.10.10.10').valid_ip?
+        refute LoadRanges.new('192.168.127.254').valid_ip?
+        assert_equal 1, LoadRanges.counter
+
+        LoadRanges.reload_ips
+        refute LoadRanges.new('127.127.127.127').valid_ip?
+        refute LoadRanges.new('10.10.10.10').valid_ip?
+        assert LoadRanges.new('192.168.127.254').valid_ip?
+        assert_equal 2, LoadRanges.counter
+      end
+      # rubocop:enable Metrics/AbcSize
+    end
+  end
+end

data/test/legitbot_test.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class LegitbotTest < Minitest::Test
   def test_rules
-    assert !Legitbot.bot("Firefox", "127.0.0.1"), msg: "Not a bot"
-    assert Legitbot.bot("Googlebot", "5.140.70.64"), msg: "No reverse resolve, bot"
+    assert !Legitbot.bot('Firefox', '127.0.0.1'),
+           msg: 'Not a bot'
+    assert Legitbot.bot('Googlebot', '5.140.70.64'),
+           msg: 'No reverse resolve, bot'
 
-    Legitbot.bot("Firefox", "127.0.0.1") do |bot|
-      flunk "No bot Firefox is possible"
+    Legitbot.bot('Firefox', '127.0.0.1') do |_bot|
+      flunk 'No bot Firefox is possible'
     end
   end
 end

data/test/oracle_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class OracleTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Oracle.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Oracle IP"
+  end
+
+  def test_valid_ip
+    ip = '148.64.56.64'
+    match = Legitbot::Oracle.new ip
+    assert match.valid?, msg: "#{ip} is a valid Oracle IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Oracle detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Oracle'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
+      '148.64.56.64'
+    )
+    assert bot, msg: 'Oracle detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Oracle'
+  end
+end

data/test/pinterest_test.rb

@@ -1,42 +1,54 @@
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'legitbot'
 
 class PinterestTest < Minitest::Test
   def test_malicious_ip
-    ip = "149.210.164.47"
+    ip = '149.210.164.47'
     match = Legitbot::Pinterest.new ip
-    reverse_name = match.reverse_name
-    assert !match.subdomain_of?("pinterest.com."), msg: "#{reverse_name} is not a subdomain of pinterest.com"
     assert !match.valid?, msg: "#{ip} is not a real Pinterest IP"
   end
 
   def test_valid_ip
-    ip = "54.236.1.11"
+    ip = '54.236.1.11'
     match = Legitbot::Pinterest.new ip
-    reverse_name = match.reverse_name
-    assert match.subdomain_of?("pinterest.com."), msg: "#{reverse_name} is a subdomain of pinterest.com"
     assert match.valid?, msg: "#{ip} is a valid Pinterest IP"
   end
 
   def test_malicious_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "149.210.164.47")
-    assert bot, msg: "Pinterest detected from User-Agent"
-    assert !bot.valid?, msg: "Not a valid Pinterest"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Pinterest detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Pinterest'
   end
 
   def test_valid_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "54.236.1.11")
-    assert bot, msg: "Pinterest detected from User-Agent"
-    assert bot.valid?, msg: "Valid Pinterest"
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
+      '54.236.1.11'
+    )
+    assert bot, msg: 'Pinterest detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Pinterest'
   end
 
+  # rubocop:disable Metrics/LineLength
   def test_android_not_bot
-    bot = Legitbot.bot("Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]", "85.117.106.133")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]',
+      '85.117.106.133'
+    )
     assert_nil bot
   end
+  # rubocop:enable Metrics/LineLength
 
   def test_engine_name
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "54.236.1.11")
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
+      '54.236.1.11'
+    )
     assert_equal :pinterest, bot.detected_as
   end
 end

data/test/twitter_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class TwitterTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Twitter.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Twitter IP"
+  end
+
+  def test_valid_ip
+    ip = '199.16.156.125'
+    match = Legitbot::Twitter.new ip
+    assert match.valid?, msg: "#{ip} is a valid Twitter IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Twitterbot/1.0',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Twitter detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Twitter'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Twitterbot/1.0',
+      '199.16.156.125'
+    )
+    assert bot, msg: 'Twitter detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Twitter'
+  end
+end