legitbot 1.5.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90eab3a4d53cc388fac2db75058e11c549ec09b3acb3352a398c10e437c0f233
-  data.tar.gz: 5a252a30d37a44de60bb5b7baf1f264477211c22c0acc1477317f8281b7e9103
+  metadata.gz: a835b0a1636ec6a95b61ac14b299f480f48b251678bdfd3228769ba1f36d4160
+  data.tar.gz: 8d548e2d5584d4239f9d1f758a09fdf40f7d825cc31b74add702cea52438e4a8
 SHA512:
-  metadata.gz: c56a32b2bccb2667f5c42c7580c31b2d79884331dfecf5f3cd01f1242ad2e53a5a4d1192a01f1f814014ef3da38702ad5b6041da3ba89d6b14cfbb5658550c47
-  data.tar.gz: c388aad0dec6a86a8d7a9ae23ae1e3200cd7eb3df421f74934b1a7582d9e63c4da1b66d9f28def1685f821c9ac2813b2d7fa38c4653a0a72301576c1cdddcf9b
+  metadata.gz: 4bca3fd8614ff5bd73d733573dbf0ff23ab233682e2e7eb054638eb963b3d68ced441f0ed8faac418f6af125011da07b8ba767ed5819fcbcc571ae1c6253ab51
+  data.tar.gz: d3ef89deb8d0fd22915bbb503665ed7f805bbfa2bc61396f04792d5321b478a8eca79b5df4c9ec39a8bb6deec608711cec32f279fa155544f072e8d4e921c1a3

data/.github/workflows/autocorrect.yml

@@ -0,0 +1,44 @@
+name: autocorrect
+
+on:
+  schedule:
+  - cron: '29 6 * * 6'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ 3.0 ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v3
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run tests
+      run: bundle exec rake test
+    - name: Run linter
+      run: bundle exec rubocop --auto-correct
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v3
+      with:
+        branch: update/lint-autocorrect
+        delete-branch: true
+        commit-message: '🔧 autocorrect by Rubocop'
+        title: '[rubocop] Automatic corrections'
+        body: |
+          Please review automatic changes proposed by `rubocop -a`.

data/.github/workflows/build.yml

@@ -3,8 +3,6 @@ name: build
 on:
   pull_request:
   push:
-  schedule:
-  - cron: '29 6 * * 6'
 
 jobs:
   test:
@@ -13,16 +11,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ jruby, 2.5, 2.6, 2.7 ]
+        ruby: [ jruby, 2.6, 2.7, 3.0 ]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Cache dependencies
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       with:
         path: vendor/bundle
         key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
@@ -41,16 +39,16 @@ jobs:
 
     strategy:
       matrix:
-        ruby: [ 2.7 ]
+        ruby: [ 3.0 ]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Cache dependencies
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       with:
         path: vendor/bundle
         key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}

data/.rubocop.yml

@@ -1,3 +1,7 @@
+require:
+  - rubocop-minitest
+  - rubocop/cop/custom
+
 AllCops:
   CacheRootDirectory: 'vendor'
   NewCops: enable
@@ -11,3 +15,6 @@ Naming/MemoizedInstanceVariableName:
 
 Style/MapToHash:
   Enabled: false
+
+Custom/IpRanges:
+  Enabled: true

data/.ruby-version

@@ -1 +1 @@
-2.7.3
+3.0.3

data/README.md

@@ -62,6 +62,7 @@ end
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
 * [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
+* [You.com](https://about.you.com/youbot/)
 
 ## License
 

data/Rakefile

@@ -1,25 +1 @@
 # frozen_string_literal: true
-
-require 'rubygems'
-require 'bundler'
-require 'bump/tasks'
-require 'rake/testtask'
-Bundler::GemHelper.install_tasks
-
-Bump.tag_by_default = true
-
-Rake::TestTask.new do |t|
-  t.libs << 'test'
-  t.test_files = FileList['test/**/*_test.rb']
-  t.warning = true
-  t.verbose = true
-end
-
-desc 'Start a console'
-task :console do
-  require 'irb'
-  ARGV.clear
-  IRB.start
-end
-
-task default: %w[test]

data/legitbot.gemspec

@@ -19,15 +19,18 @@ Gem::Specification.new do |spec|
     'rubygems_mfa_required' => 'true'
   }
 
-  spec.required_ruby_version = '>= 2.5.0'
-  spec.add_dependency 'augmented_interval_tree', '~> 0.1', '>= 0.1.1'
+  spec.required_ruby_version = '>= 2.6.0'
+  spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
   spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
+  spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
+  spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
+  spec.add_development_dependency 'nokogiri', '~> 1.13.0', '>= 1.13.0'
   spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.24.0', '>= 1.24.0'
+  spec.add_development_dependency 'rubocop', '~> 1.31.0', '>= 1.31.0'
+  spec.add_development_dependency 'rubocop-minitest', '~> 0.20.0', '>= 0.20.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']
-  spec.test_files = Dir.glob('test/**/*')
 end

data/lib/legitbot/duckduckgo.rb

@@ -3,21 +3,21 @@
 module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
+    # @fetch:url https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
+    # @fetch:selector section.main article.content ul > li
     ip_ranges %w[
-      20.191.45.212
-      40.88.21.235
-      40.76.173.151
-      40.76.163.7
-      20.185.79.47
-      52.142.26.175
       20.185.79.15
-      52.142.24.149
-      40.76.162.208
-      40.76.163.23
+      20.185.79.47
+      20.191.45.212
       40.76.162.191
+      40.76.162.208
       40.76.162.247
-      54.208.102.37
-      107.21.1.8
+      40.76.163.7
+      40.76.163.23
+      40.76.173.151
+      40.88.21.235
+      52.142.24.149
+      52.142.26.175
     ]
   end
 

data/lib/legitbot/validators/domains.rb

@@ -64,6 +64,8 @@ module Legitbot
           return nil if record.nil?
 
           resolver.getaddress(record.to_s).to_s
+        rescue Resolv::ResolvError
+          nil
         end
       end
     end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.5.1'
+  VERSION = '1.7.0'
 end

data/lib/legitbot/you.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://about.you.com/youbot/
+  class You < BotMatch
+    ip_ranges %w[
+      20.59.40.22
+    ]
+  end
+
+  rule Legitbot::You, %w[YouBot/]
+end

data/lib/legitbot.rb

@@ -13,7 +13,8 @@ require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
 require_relative 'legitbot/oracle'
+require_relative 'legitbot/petalbot'
 require_relative 'legitbot/pinterest'
 require_relative 'legitbot/twitter'
 require_relative 'legitbot/yandex'
-require_relative 'legitbot/petalbot'
+require_relative 'legitbot/you'

data/lib/rubocop/cop/custom/ip_ranges.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'net/http'
+require 'nokogiri'
+require 'rubocop'
+require 'uri'
+
+module RuboCop
+  module Cop
+    module Custom
+      class IpRanges < Base # :nodoc:
+        extend AutoCorrector
+
+        MSG = 'Outdated list of IP ranges compared to %<url>s'
+        REGEXP = /^\s*#\s*@fetch:(?<param>[a-z0-9_]+)\s+(?<arg>.*)?/
+
+        def_node_matcher :on_ip_ranges, <<~PATTERN
+          (send nil? :ip_ranges $(array str+))
+        PATTERN
+
+        def on_send(node)
+          on_ip_ranges(node) do |value|
+            params = fetch_params(node)
+            return unless mandatory_params?(params)
+
+            existing_ips = read_node_ips value
+            new_ips = fetch_ips(**params)
+            return if existing_ips == new_ips
+
+            register_offense(value, new_ips, **params)
+          end
+        end
+
+        private
+
+        def fetch_ips(url:, selector:)
+          response = Net::HTTP.get_response URI(url)
+          response.value
+
+          document = Nokogiri::HTML response.body
+          document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
+        end
+
+        def read_node_ips(value)
+          value.child_nodes.map(&:value).sort_by(&IPAddr.method(:new))
+        end
+
+        def register_offense(node, new_ips, **params)
+          message = format(MSG, params)
+          add_offense node, message: message do |corrector|
+            corrector.replace node, node_replacement(new_ips)
+          end
+        end
+
+        def mandatory_params?(params)
+          params.include?(:url) && params.include?(:selector)
+        end
+
+        def fetch_params(node)
+          comments = processed_source.ast_with_comments[node]
+          comments.map do |comment|
+            match = comment.text.match(REGEXP)
+            next unless match
+
+            [match[:param].to_sym, match[:arg]]
+          end.compact.to_h
+        end
+
+        def node_replacement(new_ips)
+          contents = new_ips.join("\n")
+          "%w[\n#{contents}\n]"
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/cop/custom.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'custom/ip_ranges'

data/rakelib/bump.rake

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'bump/tasks'
+
+Bump.tag_by_default = true

data/rakelib/bundler.rake

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/rakelib/console.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+desc 'Start a console'
+task :console do
+  require 'irb'
+  ARGV.clear
+  IRB.start
+end

data/rakelib/test.rake

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.warning = true
+  t.verbose = true
+end
+
+task default: %w[test]

data/test/ahrefs_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class AhrefsTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Ahrefs.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Ahrefs IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.36.148.0'
     match = Legitbot::Ahrefs.new ip
-    assert match.valid?, msg: "#{ip} is a valid Ahrefs IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +26,8 @@ class AhrefsTest < Minitest::Test
       'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Ahrefs detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Ahrefs'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,7 +35,7 @@ class AhrefsTest < Minitest::Test
       'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
       '54.36.148.0'
     )
-    assert bot, msg: 'Ahrefs detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Ahrefs'
+    assert bot
+    assert_predicate bot, :valid?
   end
 end

data/test/alexa_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class AlexaTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Alexa.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Alexa IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '52.86.176.3'
     match = Legitbot::Alexa.new ip
-    assert match.valid?, msg: "#{ip} is a valid Alexa IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +26,8 @@ class AlexaTest < Minitest::Test
       'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Alexa detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Alexa'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,7 +35,7 @@ class AlexaTest < Minitest::Test
       'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
       '52.86.176.3'
     )
-    assert bot, msg: 'Alexa detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Alexa'
+    assert bot
+    assert_predicate bot, :valid?
   end
 end

data/test/amazon_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class AmazonTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Amazon.new ip
-    assert !match.valid?, msg: "#{ip} is not a real AmazonAdBot IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.166.7.90'
     match = Legitbot::Amazon.new ip
-    assert match.valid?, msg: "#{ip} is a valid AmazonAdBot IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +26,8 @@ class AmazonTest < Minitest::Test
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '149.210.164.47'
     )
-    assert bot, msg: 'AmazonAdBot detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid AmazonAdBot'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,8 +35,8 @@ class AmazonTest < Minitest::Test
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '54.166.7.90'
     )
-    assert bot, msg: 'AmazonAdBot detected from User-Agent'
-    assert bot.valid?, msg: 'Valid AmazonAdBot'
+    assert bot
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/apple_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class AppleTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_valid_ip
     ip = '17.58.98.60'
     match = Legitbot::Apple.new(ip)
-    assert match.valid?, msg: "#{ip} is a valid Applebot IP"
+    assert_predicate match, :valid?
   end
 
   def test_invalid_ip
     ip = '127.0.0.1'
     match = Legitbot::Apple.new(ip)
-    assert match.fake?, msg: "#{ip} is a fake Applebot IP"
+    assert_predicate match, :fake?
   end
 
   # rubocop:disable Layout/LineLength
@@ -23,7 +28,7 @@ class AppleTest < Minitest::Test
       '17.58.98.60'
     )
     assert_equal :apple, bot.detected_as
-    assert bot.valid?, msg: 'A valid Applebot User-agent and IP'
+    assert_predicate bot, :valid?
   end
   # rubocop:enable Layout/LineLength
 end

data/test/botmatch_test.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class BotMatchTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_valid_class_syntax
-    assert Legitbot::Google.valid?('66.249.64.141'), msg: 'Valid Googlebot'
-    assert Legitbot::Google.fake?('149.210.164.47'), msg: 'Fake Googlebot'
+    assert Legitbot::Google.valid?('66.249.64.141')
+    assert Legitbot::Google.fake?('149.210.164.47')
   end
 end

data/test/facebook_test.rb

@@ -20,44 +20,48 @@ class FacebookTest < Minitest::Test
   def test_valid_ip
     ip = '69.63.186.89'
     match = Legitbot::Facebook.new(ip)
-    assert match.valid?, msg: "#{ip} is a valid Facebook IP"
+    assert_predicate match, :valid?
 
     ip = '69.171.251.1'
     match = Legitbot::Facebook.new(ip)
-    assert match.valid?, msg: "#{ip} is a valid Facebook IP"
+    assert_predicate match, :valid?
   end
 
   def test_invalid_ip
     ip = '127.0.0.1'
     match = Legitbot::Facebook.new(ip)
-    assert match.fake?, msg: "#{ip} is a fake Facebook IP"
+    assert_predicate match, :fake?
   end
 
-  # rubocop:disable Layout/LineLength, Metrics/MethodLength
-  def test_user_agent
+  def test_user_agent1
     Legitbot.bot(
       'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
       '31.13.76.56'
     ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?, msg: 'true Facebook'
+      assert_predicate bot, :valid?
     end
+  end
 
+  def test_user_agent2
     Legitbot.bot(
       'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
       '173.252.87.8'
     ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?, msg: 'true Facebook'
+      assert_predicate bot, :valid?
     end
+  end
 
+  # rubocop:disable Layout/LineLength
+  def test_user_agent3
     Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
       '92.243.181.7'
     ) do |bot|
-      assert %i[facebook twitter].include?(bot.detected_as)
-      assert bot.fake?, msg: 'fake Facebook'
+      assert_includes %i[facebook twitter], bot.detected_as
+      assert_predicate bot, :fake?
     end
   end
-  # rubocop:enable Layout/LineLength, Metrics/MethodLength
+  # rubocop:enable Layout/LineLength
 end

data/test/google_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class GoogleTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Google.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Googlebot IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '66.249.64.141'
     match = Legitbot::Google.new ip
-    assert match.valid?, msg: "#{ip} is a valid Googlebot IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +26,8 @@ class GoogleTest < Minitest::Test
       'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Googlebot detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Googlebot'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,8 +35,8 @@ class GoogleTest < Minitest::Test
       'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
       '66.249.64.141'
     )
-    assert bot, msg: 'Googlebot detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Googlebot'
+    assert bot
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/legitbot/validators/ip_ranges_test.rb

@@ -4,6 +4,7 @@ require 'minitest/autorun'
 require 'legitbot'
 
 module Legitbot
+  # rubocop:disable Minitest/MultipleAssertions
   module Validators
     class NoRanges
       include IpRanges
@@ -86,7 +87,7 @@ module Legitbot
         assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
 
         matcher = EmptyRanges.new '127.0.0.0'
-        assert matcher.valid_ip?
+        assert_predicate matcher, :valid_ip?
       end
 
       def test_matcher_array
@@ -106,15 +107,15 @@ module Legitbot
       # rubocop:disable Metrics/AbcSize
       def test_matcher_loader
         assert_equal 0, LoadRanges.counter
-        assert LoadRanges.new('127.127.127.127').valid_ip?
-        refute LoadRanges.new('10.10.10.10').valid_ip?
-        refute LoadRanges.new('192.168.127.254').valid_ip?
+        assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
+        refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
+        refute_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
         assert_equal 1, LoadRanges.counter
 
         LoadRanges.reload_ips
-        refute LoadRanges.new('127.127.127.127').valid_ip?
-        refute LoadRanges.new('10.10.10.10').valid_ip?
-        assert LoadRanges.new('192.168.127.254').valid_ip?
+        refute_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
+        refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
+        assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
         assert_equal 2, LoadRanges.counter
       end
       # rubocop:enable Metrics/AbcSize
@@ -128,4 +129,5 @@ module Legitbot
       end
     end
   end
+  # rubocop:enable Minitest/MultipleAssertions
 end

data/test/legitbot_test.rb

@@ -5,10 +5,8 @@ require 'legitbot'
 
 class LegitbotTest < Minitest::Test
   def test_rules
-    assert !Legitbot.bot('Firefox', '127.0.0.1'),
-           msg: 'Not a bot'
-    assert Legitbot.bot('Googlebot', '5.140.70.64'),
-           msg: 'No reverse resolve, bot'
+    refute Legitbot.bot('Firefox', '127.0.0.1')
+    assert Legitbot.bot('Googlebot', '5.140.70.64')
 
     Legitbot.bot('Firefox', '127.0.0.1') do |_bot|
       flunk 'No bot Firefox is possible'

data/test/lib/dns_server_mock.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'dns_mock'
+require 'json'
+
+TEST_DNS_RECORDS = {
+  # Malicious
+  '149.210.164.47' => {
+    ptr: %w[malicious.spam.co]
+  },
+
+  # Ahrefs
+  'ip-54-36-148-0.a.ahrefs.com' => {
+    a: %w[54.36.148.0]
+  },
+  '54.36.148.0' => {
+    ptr: %w[ip-54-36-148-0.a.ahrefs.com]
+  },
+
+  # Alexa
+  '52.86.176.3' => {
+    ptr: %w[crawl-52-86-176-3.alexa.com]
+  },
+
+  # Amazon
+  'crawler-54-166-7-90.amazonadbot.com' => {
+    a: %w[54.166.7.90]
+  },
+  '54.166.7.90' => {
+    ptr: %w[crawler-54-166-7-90.amazonadbot.com]
+  },
+
+  # Apple
+  '17-58-98-60.applebot.apple.com' => {
+    a: %w[17.58.98.60]
+  },
+  '17.58.98.60' => {
+    ptr: %w[17-58-98-60.applebot.apple.com]
+  },
+
+  # Google
+  'crawl-66-249-64-141.googlebot.com' => {
+    a: %w[66.249.64.141]
+  },
+  '66.249.64.141' => {
+    ptr: %w[crawl-66-249-64-141.googlebot.com]
+  },
+
+  # Petalbot
+  'petalbot-114-119-134-10.petalsearch.com' => {
+    a: %w[114.119.134.10]
+  },
+  '114.119.134.10' => {
+    ptr: %w[petalbot-114-119-134-10.petalsearch.com]
+  },
+
+  # Pinterest
+  'crawl-54-236-1-11.pinterest.com' => {
+    a: %w[54.236.1.11]
+  },
+  '54.236.1.11' => {
+    ptr: %w[crawl-54-236-1-11.pinterest.com]
+  }
+}.freeze
+
+class DnsServer
+  class << self
+    attr_accessor :mock
+  end
+
+  @mock = DnsMock.start_server records: TEST_DNS_RECORDS
+end
+
+module DnsServerMock
+  def before_all
+    super
+
+    Legitbot.resolver_config = {
+      nameserver: 'localhost',
+      nameserver_port: [['localhost', DnsServer.mock.port]]
+    }
+  end
+
+  def after_all
+    Legitbot.resolver_config = nil
+
+    super
+  end
+end

data/test/oracle_test.rb

@@ -7,13 +7,13 @@ class OracleTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Oracle.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Oracle IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '148.64.56.64'
     match = Legitbot::Oracle.new ip
-    assert match.valid?, msg: "#{ip} is a valid Oracle IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +21,8 @@ class OracleTest < Minitest::Test
       'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Oracle detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Oracle'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,7 +30,7 @@ class OracleTest < Minitest::Test
       'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
       '148.64.56.64'
     )
-    assert bot, msg: 'Oracle detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Oracle'
+    assert bot
+    assert_predicate bot, :valid?
   end
 end

data/test/petalbot_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class PetalbotTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Petalbot.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Petalbot IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
-    ip = '114.119.128.10'
+    ip = '114.119.134.10'
     match = Legitbot::Petalbot.new ip
-    assert match.valid?, msg: "#{ip} is a valid Petalbot IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,17 +26,17 @@ class PetalbotTest < Minitest::Test
       'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Petalbot detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Petalbot'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
-      '114.119.128.10'
+      '114.119.134.10'
     )
-    assert bot, msg: 'Petalbot detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Petalbot'
+    assert bot
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/pinterest_test.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require 'minitest/autorun'
+require 'minitest/hooks/test'
+require 'lib/dns_server_mock'
 require 'legitbot'
 
 class PinterestTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Pinterest.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Pinterest IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.236.1.11'
     match = Legitbot::Pinterest.new ip
-    assert match.valid?, msg: "#{ip} is a valid Pinterest IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +26,8 @@ class PinterestTest < Minitest::Test
       'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
       '149.210.164.47'
     )
-    assert bot, msg: 'Pinterest detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Pinterest'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,8 +35,8 @@ class PinterestTest < Minitest::Test
       'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
       '54.236.1.11'
     )
-    assert bot, msg: 'Pinterest detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Pinterest'
+    assert bot
+    assert_predicate bot, :valid?
   end
 
   # rubocop:disable Layout/LineLength

data/test/twitter_test.rb

@@ -7,13 +7,13 @@ class TwitterTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Twitter.new ip
-    assert !match.valid?, msg: "#{ip} is not a real Twitter IP"
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '199.16.156.125'
     match = Legitbot::Twitter.new ip
-    assert match.valid?, msg: "#{ip} is a valid Twitter IP"
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -21,8 +21,8 @@ class TwitterTest < Minitest::Test
       'Twitterbot/1.0',
       '149.210.164.47'
     )
-    assert bot, msg: 'Twitter detected from User-Agent'
-    assert !bot.valid?, msg: 'Not a valid Twitter'
+    assert bot
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -30,7 +30,7 @@ class TwitterTest < Minitest::Test
       'Twitterbot/1.0',
       '199.16.156.125'
     )
-    assert bot, msg: 'Twitter detected from User-Agent'
-    assert bot.valid?, msg: 'Valid Twitter'
+    assert bot
+    assert_predicate bot, :valid?
   end
 end

data/test/you_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class YouTest < Minitest::Test
+  def test_malicious_ip
+    ip = '20.59.41.22'
+    match = Legitbot::You.new ip
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '20.59.40.22'
+    match = Legitbot::You.new ip
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
+      '20.59.41.22'
+    )
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
+      '20.59.40.22'
+    )
+    assert bot
+    assert_predicate bot, :valid?
+  end
+end

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-01 00:00:00.000000000 Z
+date: 2022-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: augmented_interval_tree
+  name: fast_interval_tree
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: irrc
   requirement: !ruby/object:Gem::Requirement
@@ -70,6 +70,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: dns_mock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -90,6 +110,46 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.0
+- !ruby/object:Gem::Dependency
+  name: minitest-hooks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -116,20 +176,40 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 1.31.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.31.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.31.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.31.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.20.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 0.20.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 0.20.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 0.20.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -138,6 +218,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/autocorrect.yml"
 - ".github/workflows/build.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -168,6 +249,13 @@ files:
 - lib/legitbot/validators/ip_ranges.rb
 - lib/legitbot/version.rb
 - lib/legitbot/yandex.rb
+- lib/legitbot/you.rb
+- lib/rubocop/cop/custom.rb
+- lib/rubocop/cop/custom/ip_ranges.rb
+- rakelib/bump.rake
+- rakelib/bundler.rake
+- rakelib/console.rake
+- rakelib/test.rake
 - test/ahrefs_test.rb
 - test/alexa_test.rb
 - test/amazon_test.rb
@@ -178,10 +266,12 @@ files:
 - test/legitbot/validators/domains_test.rb
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb
+- test/lib/dns_server_mock.rb
 - test/oracle_test.rb
 - test/petalbot_test.rb
 - test/pinterest_test.rb
 - test/twitter_test.rb
+- test/you_test.rb
 homepage: https://github.com/alaz/legitbot
 licenses:
 - Apache-2.0
@@ -196,29 +286,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: 'Validate requests from Web crawlers: impersonating or not?'
-test_files:
-- test/legitbot_test.rb
-- test/legitbot/validators/domains_test.rb
-- test/legitbot/validators/ip_ranges_test.rb
-- test/pinterest_test.rb
-- test/alexa_test.rb
-- test/ahrefs_test.rb
-- test/apple_test.rb
-- test/oracle_test.rb
-- test/google_test.rb
-- test/amazon_test.rb
-- test/petalbot_test.rb
-- test/botmatch_test.rb
-- test/facebook_test.rb
-- test/twitter_test.rb
+test_files: []