legitbot 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df2907ab34b159613adf21c3301607f53fa893e0823f81560e45ad2c8303075b
-  data.tar.gz: fe913e9b72878969839326da227c1169ad8234a8ae335e276f614b53e5777d42
+  metadata.gz: 0bd071c617ddb67424715ae30b92783fa0155ca182c7ef0c7cac0a5962eff8d1
+  data.tar.gz: 33ed371772200f2d27d95d083782d73d1572eeac329f63e5e3d5f14937151b72
 SHA512:
-  metadata.gz: f8af647da764cd722f27f2936f8f9728f9cdd7b313e295d9c80ebe7144a601a5360f5b615039d5afe75f7ded891e3bfc06292383e9e41e342977d39f1e485dab
-  data.tar.gz: 74e3dd5cb2c4d2e7df717c6bb3a072c74aa51b922f7f71d280cde82387ba458d30970f85c4f79508852ef5c2969196bd640427bf2323104229616960314fb9df
+  metadata.gz: d188e8a97d9fd5115897f69c31a2f43f8b55b0426bd932c347800fd023231dfbc1c6c4ada108c94cdd4258673cbee06d012182ef3e815ff7be8b010b9d4eb4db
+  data.tar.gz: a00e24aa78775c201f9587286bb7cec46ec9993687ace3e675c2e4f585ff4ff56839f9a977e9709dab13e652098e72910a6f86b4dbc5fa38083970ca9085c143

data/.github/workflows/autocorrect.yml

@@ -0,0 +1,44 @@
+name: autocorrect
+
+on:
+  schedule:
+  - cron: '29 6 * * 6'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ 3.0 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run tests
+      run: bundle exec rake test
+    - name: Run linter
+      run: bundle exec rubocop --auto-correct
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v3
+      with:
+        branch: update/lint-autocorrect
+        delete-branch: true
+        commit-message: '🔧 autocorrect by Rubocop'
+        title: '[rubocop] Automatic corrections'
+        body: |
+          Please review automatic changes proposed by `rubocop -a`.

data/.github/workflows/build.yml

@@ -3,8 +3,6 @@ name: build
 on:
   pull_request:
   push:
-  schedule:
-  - cron: '29 6 * * 6'
 
 jobs:
   test:

data/.rubocop.yml

@@ -1,5 +1,6 @@
 require:
-- rubocop-minitest
+  - rubocop-minitest
+  - rubocop/cop/custom
 
 AllCops:
   CacheRootDirectory: 'vendor'
@@ -14,3 +15,6 @@ Naming/MemoizedInstanceVariableName:
 
 Style/MapToHash:
   Enabled: false
+
+Custom/IpRanges:
+  Enabled: true

data/legitbot.gemspec

@@ -26,9 +26,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
   spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
+  spec.add_development_dependency 'nokogiri', '~> 1.13.0', '>= 1.13.0'
   spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.24.0', '>= 1.24.0'
-  spec.add_development_dependency 'rubocop-minitest', '~> 0.17.0', '>= 0.17.0'
+  spec.add_development_dependency 'rubocop', '~> 1.28.0', '>= 1.28.0'
+  spec.add_development_dependency 'rubocop-minitest', '~> 0.19.0', '>= 0.19.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/duckduckgo.rb

@@ -3,21 +3,21 @@
 module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
+    # @fetch:url https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
+    # @fetch:selector section.main article.content ul > li
     ip_ranges %w[
-      20.191.45.212
-      40.88.21.235
-      40.76.173.151
-      40.76.163.7
-      20.185.79.47
-      52.142.26.175
       20.185.79.15
-      52.142.24.149
-      40.76.162.208
-      40.76.163.23
+      20.185.79.47
+      20.191.45.212
       40.76.162.191
+      40.76.162.208
       40.76.162.247
-      54.208.102.37
-      107.21.1.8
+      40.76.163.7
+      40.76.163.23
+      40.76.173.151
+      40.88.21.235
+      52.142.24.149
+      52.142.26.175
     ]
   end
 

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.6.0'
+  VERSION = '1.6.1'
 end

data/lib/rubocop/cop/custom/ip_ranges.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'net/http'
+require 'nokogiri'
+require 'rubocop'
+require 'uri'
+
+module RuboCop
+  module Cop
+    module Custom
+      class IpRanges < Base # :nodoc:
+        extend AutoCorrector
+
+        MSG = 'Outdated list of IP ranges compared to %<url>s'
+        REGEXP = /^\s*#\s*@fetch:(?<param>[a-z0-9_]+)\s+(?<arg>.*)?/
+
+        def_node_matcher :on_ip_ranges, <<~PATTERN
+          (send nil? :ip_ranges $(array str+))
+        PATTERN
+
+        def on_send(node)
+          on_ip_ranges(node) do |value|
+            params = fetch_params(node)
+            return unless mandatory_params?(params)
+
+            existing_ips = read_node_ips value
+            new_ips = fetch_ips(**params)
+            return if existing_ips == new_ips
+
+            register_offense(value, new_ips, **params)
+          end
+        end
+
+        private
+
+        def fetch_ips(url:, selector:)
+          response = Net::HTTP.get_response URI(url)
+          response.value
+
+          document = Nokogiri::HTML response.body
+          document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
+        end
+
+        def read_node_ips(value)
+          value.child_nodes.map(&:value).sort_by(&IPAddr.method(:new))
+        end
+
+        def register_offense(node, new_ips, **params)
+          message = format(MSG, params)
+          add_offense node, message: message do |corrector|
+            corrector.replace node, node_replacement(new_ips)
+          end
+        end
+
+        def mandatory_params?(params)
+          params.include?(:url) && params.include?(:selector)
+        end
+
+        def fetch_params(node)
+          comments = processed_source.ast_with_comments[node]
+          comments.map do |comment|
+            match = comment.text.match(REGEXP)
+            next unless match
+
+            [match[:param].to_sym, match[:arg]]
+          end.compact.to_h
+        end
+
+        def node_replacement(new_ips)
+          contents = new_ips.join("\n")
+          "%w[\n#{contents}\n]"
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/cop/custom.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'custom/ip_ranges'

data/test/ahrefs_test.rb

@@ -12,13 +12,13 @@ class AhrefsTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Ahrefs.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.36.148.0'
     match = Legitbot::Ahrefs.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class AhrefsTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,6 +36,6 @@ class AhrefsTest < Minitest::Test
       '54.36.148.0'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 end

data/test/alexa_test.rb

@@ -12,13 +12,13 @@ class AlexaTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Alexa.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '52.86.176.3'
     match = Legitbot::Alexa.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class AlexaTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,6 +36,6 @@ class AlexaTest < Minitest::Test
       '52.86.176.3'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 end

data/test/amazon_test.rb

@@ -12,13 +12,13 @@ class AmazonTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Amazon.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.166.7.90'
     match = Legitbot::Amazon.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class AmazonTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,7 +36,7 @@ class AmazonTest < Minitest::Test
       '54.166.7.90'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/apple_test.rb

@@ -12,13 +12,13 @@ class AppleTest < Minitest::Test
   def test_valid_ip
     ip = '17.58.98.60'
     match = Legitbot::Apple.new(ip)
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_invalid_ip
     ip = '127.0.0.1'
     match = Legitbot::Apple.new(ip)
-    assert match.fake?
+    assert_predicate match, :fake?
   end
 
   # rubocop:disable Layout/LineLength
@@ -28,7 +28,7 @@ class AppleTest < Minitest::Test
       '17.58.98.60'
     )
     assert_equal :apple, bot.detected_as
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
   # rubocop:enable Layout/LineLength
 end

data/test/facebook_test.rb

@@ -20,17 +20,17 @@ class FacebookTest < Minitest::Test
   def test_valid_ip
     ip = '69.63.186.89'
     match = Legitbot::Facebook.new(ip)
-    assert match.valid?
+    assert_predicate match, :valid?
 
     ip = '69.171.251.1'
     match = Legitbot::Facebook.new(ip)
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_invalid_ip
     ip = '127.0.0.1'
     match = Legitbot::Facebook.new(ip)
-    assert match.fake?
+    assert_predicate match, :fake?
   end
 
   def test_user_agent1
@@ -39,7 +39,7 @@ class FacebookTest < Minitest::Test
       '31.13.76.56'
     ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?
+      assert_predicate bot, :valid?
     end
   end
 
@@ -49,7 +49,7 @@ class FacebookTest < Minitest::Test
       '173.252.87.8'
     ) do |bot|
       assert_equal :facebook, bot.detected_as
-      assert bot.valid?
+      assert_predicate bot, :valid?
     end
   end
 
@@ -60,7 +60,7 @@ class FacebookTest < Minitest::Test
       '92.243.181.7'
     ) do |bot|
       assert_includes %i[facebook twitter], bot.detected_as
-      assert bot.fake?
+      assert_predicate bot, :fake?
     end
   end
   # rubocop:enable Layout/LineLength

data/test/google_test.rb

@@ -12,13 +12,13 @@ class GoogleTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Google.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '66.249.64.141'
     match = Legitbot::Google.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class GoogleTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,7 +36,7 @@ class GoogleTest < Minitest::Test
       '66.249.64.141'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/legitbot/validators/ip_ranges_test.rb

@@ -87,7 +87,7 @@ module Legitbot
         assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
 
         matcher = EmptyRanges.new '127.0.0.0'
-        assert matcher.valid_ip?
+        assert_predicate matcher, :valid_ip?
       end
 
       def test_matcher_array
@@ -107,15 +107,15 @@ module Legitbot
       # rubocop:disable Metrics/AbcSize
       def test_matcher_loader
         assert_equal 0, LoadRanges.counter
-        assert LoadRanges.new('127.127.127.127').valid_ip?
-        refute LoadRanges.new('10.10.10.10').valid_ip?
-        refute LoadRanges.new('192.168.127.254').valid_ip?
+        assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
+        refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
+        refute_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
         assert_equal 1, LoadRanges.counter
 
         LoadRanges.reload_ips
-        refute LoadRanges.new('127.127.127.127').valid_ip?
-        refute LoadRanges.new('10.10.10.10').valid_ip?
-        assert LoadRanges.new('192.168.127.254').valid_ip?
+        refute_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
+        refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
+        assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
         assert_equal 2, LoadRanges.counter
       end
       # rubocop:enable Metrics/AbcSize

data/test/oracle_test.rb

@@ -7,13 +7,13 @@ class OracleTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Oracle.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '148.64.56.64'
     match = Legitbot::Oracle.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -22,7 +22,7 @@ class OracleTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -31,6 +31,6 @@ class OracleTest < Minitest::Test
       '148.64.56.64'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 end

data/test/petalbot_test.rb

@@ -12,13 +12,13 @@ class PetalbotTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Petalbot.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '114.119.134.10'
     match = Legitbot::Petalbot.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class PetalbotTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,7 +36,7 @@ class PetalbotTest < Minitest::Test
       '114.119.134.10'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 
   def test_valid_name

data/test/pinterest_test.rb

@@ -12,13 +12,13 @@ class PinterestTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Pinterest.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '54.236.1.11'
     match = Legitbot::Pinterest.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -27,7 +27,7 @@ class PinterestTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -36,7 +36,7 @@ class PinterestTest < Minitest::Test
       '54.236.1.11'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 
   # rubocop:disable Layout/LineLength

data/test/twitter_test.rb

@@ -7,13 +7,13 @@ class TwitterTest < Minitest::Test
   def test_malicious_ip
     ip = '149.210.164.47'
     match = Legitbot::Twitter.new ip
-    refute match.valid?
+    refute_predicate match, :valid?
   end
 
   def test_valid_ip
     ip = '199.16.156.125'
     match = Legitbot::Twitter.new ip
-    assert match.valid?
+    assert_predicate match, :valid?
   end
 
   def test_malicious_ua
@@ -22,7 +22,7 @@ class TwitterTest < Minitest::Test
       '149.210.164.47'
     )
     assert bot
-    refute bot.valid?
+    refute_predicate bot, :valid?
   end
 
   def test_valid_ua
@@ -31,6 +31,6 @@ class TwitterTest < Minitest::Test
       '199.16.156.125'
     )
     assert bot
-    assert bot.valid?
+    assert_predicate bot, :valid?
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-02 00:00:00.000000000 Z
+date: 2022-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -130,6 +130,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,40 +176,40 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 1.28.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 1.28.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 1.28.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.0
+        version: 1.28.0
 - !ruby/object:Gem::Dependency
   name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 0.19.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 0.19.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 0.19.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.0
+        version: 0.19.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -198,6 +218,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/autocorrect.yml"
 - ".github/workflows/build.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -228,6 +249,8 @@ files:
 - lib/legitbot/validators/ip_ranges.rb
 - lib/legitbot/version.rb
 - lib/legitbot/yandex.rb
+- lib/rubocop/cop/custom.rb
+- lib/rubocop/cop/custom/ip_ranges.rb
 - rakelib/bump.rake
 - rakelib/bundler.rake
 - rakelib/console.rake