legitbot 1.0.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1596ed8c3809fc3c3068f14d8bbbcf84232286fd7157fe724ba9515d195259cd
-  data.tar.gz: f0078d0404d752550adeeaa9c64954cbcec57fef3d0b37c98bc50d765c29bc1f
+  metadata.gz: d709839505242be7ed07c5f6413e607bd8eb127a011765b9f45c1a3e0bba2fbb
+  data.tar.gz: 53a19413068473299ce78887756a2ef50f59859e54d652e4314755178a30028c
 SHA512:
-  metadata.gz: 5640ae1e351bbd697325cd0bed0bdac45c3726e7b31923b7b5c8f51859ed025d4eb368d937a2b3bf9ef6c150d83418f8e019304dec3bc1b010572a4e1598c661
-  data.tar.gz: bf326da52d3adf1b2cfa3693b51cc763daef54c9b175cfb91b437b213722a0f578565db553f1240b6d6ee47d159bb418fdcc1db1f95cb14696b2bfb52e50b75d
+  metadata.gz: daaf7bbde5fae2ab3937b5e4f4cd936a61f54b80d5ec87eaca8db473977f633c993f267f6aeb101b94404119e58835878ee1433683cf613b476c6fc7f27b34b3
+  data.tar.gz: 1e51fe4ee02eb14bb0a95230e2ca3d4716a56cee199a40f3cfcd4d0999158526f151fa7915a9a3f0a21a9054ebc3862353e1a68a89057c2cf8bbcac4d082afb8

data/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+indent_style = space
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.yml]
+indent_style = space
+indent_size = 2

data/.github/workflows/build.yml

@@ -1,6 +1,6 @@
 name: build
 
-on: [push]
+on: [pull_request, push]
 
 jobs:
   test:

data/.rubocop.yml

@@ -1,2 +1,3 @@
 AllCops:
   CacheRootDirectory: 'vendor'
+  NewCops: enable

data/README.md

@@ -1,4 +1,4 @@
-# Legitbot ![](https://github.com/alaz/legitbot/workflows/build/badge.svg) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
+# Legitbot ![](https://github.com/alaz/legitbot/workflows/build/badge.svg) ![](https://badge.fury.io/rb/legitbot.svg)
 
 Ruby gem to check that an IP belongs to a bot, typically a search
 engine. This can be of help in protecting a web site from fake search
@@ -44,6 +44,7 @@ end
 ## Supported
 
 * [Ahrefs](https://ahrefs.com/robot)
+* [Alexa](https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-)
 * [Applebot](https://support.apple.com/en-us/HT204683)
 * [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 * [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
@@ -54,6 +55,7 @@ end
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
 * [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
+* [Petal robots (Huawei search)](http://aspiegel.com/petalbot)
 
 ## License
 

data/legitbot.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
   spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
-  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.74', '>= 0.74.0'
+  spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
+  spec.add_development_dependency 'rubocop', '~> 0.92.0', '>= 0.92.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot.rb

@@ -4,6 +4,7 @@ require_relative 'legitbot/legitbot'
 require_relative 'legitbot/botmatch'
 
 require_relative 'legitbot/ahrefs'
+require_relative 'legitbot/alexa'
 require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'
@@ -14,3 +15,4 @@ require_relative 'legitbot/oracle'
 require_relative 'legitbot/pinterest'
 require_relative 'legitbot/twitter'
 require_relative 'legitbot/yandex'
+require_relative 'legitbot/petalbot'

data/lib/legitbot/alexa.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-
+  # https://support.alexa.com/hc/en-us/articles/200462340
+  # https://support.alexa.com/hc/en-us/articles/200450194
+  class Alexa < BotMatch
+    ip_ranges %w[
+      52.86.176.3
+      52.4.48.181
+      52.2.182.169
+      52.86.185.29
+    ]
+  end
+
+  rule Legitbot::Alexa, %w[Alexabot ia_archiver]
+end

data/lib/legitbot/apple.rb

@@ -8,13 +8,5 @@ module Legitbot # :nodoc:
     ip_ranges '17.0.0.0/8'
   end
 
-  # https://support.apple.com/en-us/HT204683
-  # rubocop:disable Naming/ClassAndModuleCamelCase
-  class Apple_as_Google < BotMatch
-    ip_ranges '17.0.0.0/8'
-  end
-  # rubocop:enable Naming/ClassAndModuleCamelCase
-
   rule Legitbot::Apple, %w[Applebot]
-  rule Legitbot::Apple_as_Google, %w[Googlebot]
 end

data/lib/legitbot/legitbot.rb

@@ -18,7 +18,7 @@ module Legitbot
   # otherwise.
   # :yields: a found bot
   #
-  # rubocop:disable Metrics/CyclomaticComplexity
+  # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
   def self.bot(user_agent, ip)
     bots = @rules
            .select { |rule| rule[:fragments].any? { |f| user_agent.index f } }
@@ -33,7 +33,7 @@ module Legitbot
       selected
     end
   end
-  # rubocop:enable Metrics/CyclomaticComplexity
+  # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
   def self.rule(clazz, fragments)
     @rules << { class: clazz, fragments: fragments }

data/lib/legitbot/petalbot.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # http://aspiegel.com/petalbot
+  class Petalbot < BotMatch
+    domains 'aspiegel.com.'
+  end
+
+  rule Legitbot::Petalbot, %w[PetalBot]
+end

data/lib/legitbot/validators/ip_ranges.rb

@@ -40,7 +40,7 @@ module Legitbot
 
           obj = IPAddr.new(ip)
           ranges = valid_ips[obj.ipv4? ? :ipv4 : :ipv6].search(obj.to_i)
-          !ranges.empty?
+          !ranges.nil? && !ranges.empty?
         end
 
         def valid_ips

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.0.1'
+  VERSION = '1.3.0'
 end

data/test/alexa_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class AlexaTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Alexa.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Alexa IP"
+  end
+
+  def test_valid_ip
+    ip = '52.86.176.3'
+    match = Legitbot::Alexa.new ip
+    assert match.valid?, msg: "#{ip} is a valid Alexa IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Alexa detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Alexa'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
+      '52.86.176.3'
+    )
+    assert bot, msg: 'Alexa detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Alexa'
+  end
+end

data/test/facebook_test.rb

@@ -33,7 +33,7 @@ class FacebookTest < Minitest::Test
     assert match.fake?, msg: "#{ip} is a fake Facebook IP"
   end
 
-  # rubocop:disable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
+  # rubocop:disable Layout/LineLength, Metrics/MethodLength
   def test_user_agent
     Legitbot.bot(
       'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
@@ -59,5 +59,5 @@ class FacebookTest < Minitest::Test
       assert bot.fake?, msg: 'fake Facebook'
     end
   end
-  # rubocop:enable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
+  # rubocop:enable Layout/LineLength, Metrics/MethodLength
 end

data/test/legitbot/validators/ip_ranges_test.rb

@@ -51,6 +51,11 @@ module Legitbot
       ip_ranges { nil }
     end
 
+    class Ipv4Ranges
+      include IpRanges
+      ip_ranges { ['66.220.144.0/21'] }
+    end
+
     class IpRangesTest < Minitest::Test
       def test_partition_method
         empty = NoRanges.partition_ips([])
@@ -117,6 +122,10 @@ module Legitbot
       def test_nil_ranges
         assert NilRanges.valid_ip?('127.0.0.1')
       end
+
+      def test_ipv4_only_ranges
+        refute Ipv4Ranges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
+      end
     end
   end
 end

data/test/petalbot_test.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class PetalbotTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Petalbot.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Petalbot IP"
+  end
+
+  def test_valid_ip
+    ip = '114.119.153.50'
+    match = Legitbot::Petalbot.new ip
+    assert match.valid?, msg: "#{ip} is a valid Petalbot IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Petalbot detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Petalbot'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '114.119.153.50'
+    )
+    assert bot, msg: 'Petalbot detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Petalbot'
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '66.249.64.141'
+    )
+    assert_equal :petalbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; PetalBot/2.1; +http://www.google.com/bot.html)',
+      '81.1.172.108'
+    )
+    assert_equal :petalbot, bot.detected_as
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: augmented_interval_tree
@@ -96,40 +96,40 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: 13.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: 13.0.0
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: 0.92.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.74.0
+        version: 0.92.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: 0.92.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.74.0
+        version: 0.92.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -137,6 +137,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".github/workflows/build.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -148,6 +149,7 @@ files:
 - legitbot.gemspec
 - lib/legitbot.rb
 - lib/legitbot/ahrefs.rb
+- lib/legitbot/alexa.rb
 - lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
@@ -158,6 +160,7 @@ files:
 - lib/legitbot/google.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/oracle.rb
+- lib/legitbot/petalbot.rb
 - lib/legitbot/pinterest.rb
 - lib/legitbot/twitter.rb
 - lib/legitbot/validators/domains.rb
@@ -165,7 +168,7 @@ files:
 - lib/legitbot/version.rb
 - lib/legitbot/yandex.rb
 - test/ahrefs_test.rb
-- test/apple_as_google_test.rb
+- test/alexa_test.rb
 - test/apple_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
@@ -174,6 +177,7 @@ files:
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb
 - test/oracle_test.rb
+- test/petalbot_test.rb
 - test/pinterest_test.rb
 - test/twitter_test.rb
 homepage: https://github.com/alaz/legitbot
@@ -196,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: 'Validate requests from Web crawlers: impersonating or not?'
@@ -205,11 +209,12 @@ test_files:
 - test/legitbot/validators/domains_test.rb
 - test/legitbot/validators/ip_ranges_test.rb
 - test/pinterest_test.rb
+- test/alexa_test.rb
 - test/ahrefs_test.rb
 - test/apple_test.rb
-- test/apple_as_google_test.rb
 - test/oracle_test.rb
 - test/google_test.rb
+- test/petalbot_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
 - test/twitter_test.rb

data/test/apple_as_google_test.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require 'minitest/autorun'
-require 'legitbot'
-
-class AppleAsGoogleTest < Minitest::Test
-  def test_valid_ip
-    ip = '17.58.98.60'
-    match = Legitbot::Apple_as_Google.new(ip)
-    assert match.valid?, msg: "#{ip} is a valid Applebot IP"
-  end
-
-  def test_invalid_ip
-    ip = '127.0.0.1'
-    match = Legitbot::Apple_as_Google.new(ip)
-    assert match.fake?, msg: "#{ip} is a fake Applebot IP"
-  end
-
-  def test_user_agent
-    bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
-      '17.58.98.60'
-    )
-    assert_equal :apple_as_google, bot.detected_as
-    assert bot.valid?, msg: 'A valid Applebot User-agent and IP'
-  end
-end