legitbot 1.4.0 → 1.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 783bc47b1bc8f80e5d4818f150663a2cb6fb90e5dd2378bae456eb2fb1d78823
-  data.tar.gz: 0e274e1eaa295ffa0e00a8d6e2884194ed53108da9ecb75c3ef2ef4746048c9e
+  metadata.gz: fcbd5f9d7bbd29804b84b01d95765a0e950c3c1d7c42c78b8d2ba4e0c98ca679
+  data.tar.gz: 3f69f1ebc410d1d6db1c20e0ff489ba552ba34ffad3bbe9b55f7e69377b72f52
 SHA512:
-  metadata.gz: ec7c2f2cb78cc8a586a35bea72e39b12835679089f4b4d5cba754aa0caccee0c9e8c1c94461a7fd8fcb1cf4f8195f196346ea4abe7bd96a080c5c831f9d68c7a
-  data.tar.gz: a92ea8ff049127871988e8dbad7cb76b3c53f13dde97a157509e9013898c1da2cc3e573ae695d9af2e82372bf478348aeca6ec77a673798b52ac1e3ab623e214
+  metadata.gz: 998f9f6f722b9eb9ba57774ce3525ca139789321d0cdd5e024f954ef13d002ac839aca8de53917dda8c3fcea08d23add43ff1dd856cf54b8d68d57cefc26bb42
+  data.tar.gz: 3a539ba08618ecbfe3d93388b0046c0331654f87bf72eef23fca1594bedf8cdad35aa8eca91121f675f6e39e60b020bec5a1d19dee55c245e4e9b20fafee4ee8

data/README.md

@@ -1,13 +1,11 @@
 # Legitbot ![](https://github.com/alaz/legitbot/workflows/build/badge.svg) ![](https://badge.fury.io/rb/legitbot.svg)
 
-Ruby gem to check that an IP belongs to a bot, typically a search
-engine. This can be of help in protecting a web site from fake search
-engines.
+Ruby gem to make sure that an IP really belongs to a bot, typically a search
+engine.
 
 ## Usage
 
-Suppose you have a Web request and you'd like to make sure it's not from a fake
-search engine:
+Suppose you have a Web request and you would like to check it is not diguised:
 
 ```ruby
 bot = Legitbot.bot(userAgent, ip)
@@ -22,7 +20,7 @@ bot.valid? # => true
 bot.fake? # => false
 ```
 
-Sometimes you already know what search engine to expect. For example, you might
+Sometimes you already know which search engine to expect. For example, you might
 be using [rack-attack](https://github.com/kickstarter/rack-attack):
 
 ```ruby
@@ -31,8 +29,8 @@ Rack::Attack.blocklist("fake Googlebot") do |req|
 end
 ```
 
-Or if you do not like all these nasty crawlers stealing your content or
-maybe evaluating it and getting ready to invade your site with spammers,
+Or if you do not like all those ghoulish crawlers stealing your
+content, evaluating it and getting ready to invade your site with spammers,
 then block them all:
 
 ```ruby
@@ -52,6 +50,7 @@ end
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 * [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
+* [Petal search engine](http://aspiegel.com/petalbot)
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
 * [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)

data/legitbot.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
   spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.15.0', '>= 1.15.0'
+  spec.add_development_dependency 'rubocop', '~> 1.20.0', '>= 1.20.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/duckduckgo.rb

@@ -4,14 +4,15 @@ module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
     ip_ranges %w[
+      20.191.45.212
       23.21.227.69
       40.88.21.235
       50.16.241.113
       50.16.241.114
       50.16.241.117
       50.16.247.234
-      52.204.97.54
       52.5.190.19
+      52.204.97.54
       54.197.234.188
       54.208.100.253
       54.208.102.37

data/lib/legitbot/facebook.rb

@@ -18,5 +18,8 @@ module Legitbot # :nodoc:
     end
   end
 
-  rule Legitbot::Facebook, %w[Facebot facebookexternalhit/1.1]
+  rule Legitbot::Facebook, %w[
+    facebookexternalhit/1.1
+    facebookcatalog/1.0
+  ]
 end

data/lib/legitbot/oracle.rb

@@ -3,7 +3,31 @@
 module Legitbot # :nodoc:
   # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
   class Oracle < BotMatch
-    ip_ranges '148.64.56.0/24'
+    ip_ranges %w[
+      132.145.9.5
+      132.145.11.125
+      132.145.14.70
+      132.145.15.209
+      132.145.64.33
+      132.145.66.116
+      132.145.66.156
+      132.145.67.248
+      140.238.81.78
+      140.238.83.181
+      140.238.94.137
+      140.238.95.47
+      140.238.95.199
+      152.67.128.219
+      152.67.137.35
+      152.67.138.180
+
+      148.64.56.64/28
+      148.64.56.79
+      148.64.56.80
+      148.64.56.112/28
+      148.64.56.127
+      148.64.56.128
+    ]
   end
 
   rule Legitbot::Oracle, %w[GrapeshotCrawler]

data/lib/legitbot/petalbot.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # http://aspiegel.com/petalbot
+  # https://webmaster.petalsearch.com/site/petalbot
+  class Petalbot < BotMatch
+    domains 'petalsearch.com.'
+  end
+
+  rule Legitbot::Petalbot, %w[PetalBot]
+end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.4.0'
+  VERSION = '1.4.4'
 end

data/lib/legitbot.rb

@@ -15,3 +15,4 @@ require_relative 'legitbot/oracle'
 require_relative 'legitbot/pinterest'
 require_relative 'legitbot/twitter'
 require_relative 'legitbot/yandex'
+require_relative 'legitbot/petalbot'

data/test/petalbot_test.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class PetalbotTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Petalbot.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Petalbot IP"
+  end
+
+  def test_valid_ip
+    ip = '114.119.128.10'
+    match = Legitbot::Petalbot.new ip
+    assert match.valid?, msg: "#{ip} is a valid Petalbot IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Petalbot detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Petalbot'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '114.119.128.10'
+    )
+    assert bot, msg: 'Petalbot detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Petalbot'
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
+      '66.249.64.141'
+    )
+    assert_equal :petalbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; PetalBot/2.1; +http://www.google.com/bot.html)',
+      '81.1.172.108'
+    )
+    assert_equal :petalbot, bot.detected_as
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.4
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-28 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: augmented_interval_tree
@@ -116,20 +116,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.20.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.20.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.20.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: 1.20.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -160,6 +160,7 @@ files:
 - lib/legitbot/google.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/oracle.rb
+- lib/legitbot/petalbot.rb
 - lib/legitbot/pinterest.rb
 - lib/legitbot/twitter.rb
 - lib/legitbot/validators/domains.rb
@@ -176,6 +177,7 @@ files:
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb
 - test/oracle_test.rb
+- test/petalbot_test.rb
 - test/pinterest_test.rb
 - test/twitter_test.rb
 homepage: https://github.com/alaz/legitbot
@@ -212,6 +214,7 @@ test_files:
 - test/apple_test.rb
 - test/oracle_test.rb
 - test/google_test.rb
+- test/petalbot_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
 - test/twitter_test.rb