legitbot 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48fa3cd6d810e32c24c3027b57879438c658b64bdf2dfae3fbbcfd5ac67fb790
-  data.tar.gz: f8b77b3571978137339b06e633c9725ee8d1f4f6793bcdae8321a16cb9b37dfd
+  metadata.gz: 8d80bbf2deb17b90d29f6d19b5405b63e52911e5fadf05a2b6c10a7a8772afa6
+  data.tar.gz: fcac15d55a5f59d8076eeab89b67c07466c15f646b9205d8fdc49b29f55084b4
 SHA512:
-  metadata.gz: ff62eb5208ee4e565504ada36c3d94129171f2487c5cf61e3cfbc5388a189ab5e5645537d311a01376e16abd486d69c465d148d6a4ebf35f5cf5d4c51a212bd8
-  data.tar.gz: c3a9d54b1add735629ac5caaf744c8bb75d2d9fa88fe5e793bd351e860cf00ef39cb084eb3bb048fe34b1ab9d95dec66eb08b075b9ee0bd7657e0160873f24bb
+  metadata.gz: d0e367f8e02357527a46112cfd9acdc3448fc99320d6c0a117af81b8bd17a58c997a678747f1bc137fe7772b2c2e80fbf1fb4bb39cbdf52d6e53a4f4ce7dc1b7
+  data.tar.gz: '019f70211891e34b1290565e006ca8e58b25db9303db94a274ca8697871420f22983642f125e38faab02a4df9984bd097e0e61ee88adf0ce70940334d70232c4'

data/README.md

@@ -39,10 +39,18 @@ Rack::Attack.blocklist 'fake search engines' do |request|
 end
 ```
 
+## Versioning
+
+[Semantic versioning](https://semver.org/) with the following clarifications:
+
+* MINOR version is incremented when support for new bots is added.
+* PATCH version is incremented when validation logic for a bot changes (IP list updated, for example).
+
 ## Supported
 
 * [Ahrefs](https://ahrefs.com/robot)
 * [Alexa](https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-)
+* [Amazon AdBot](https://adbot.amazon.com/index.html)
 * [Applebot](https://support.apple.com/en-us/HT204683)
 * [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 * [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
@@ -50,10 +58,10 @@ end
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 * [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
+* [Petal search engine](http://aspiegel.com/petalbot)
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
 * [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
-* [Petal search engine](http://aspiegel.com/petalbot)
 
 ## License
 

data/legitbot.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
   spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.18.0', '>= 1.18.0'
+  spec.add_development_dependency 'rubocop', '~> 1.22.0', '>= 1.22.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/amazon.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://adbot.amazon.com/index.html
+  class Amazon < BotMatch
+    domains 'amazonadbot.com.'
+  end
+
+  rule Legitbot::Amazon, %w[AmazonAdBot]
+end

data/lib/legitbot/facebook.rb

@@ -18,5 +18,8 @@ module Legitbot # :nodoc:
     end
   end
 
-  rule Legitbot::Facebook, %w[Facebot facebookexternalhit/1.1]
+  rule Legitbot::Facebook, %w[
+    facebookexternalhit/1.1
+    facebookcatalog/1.0
+  ]
 end

data/lib/legitbot/google.rb

@@ -7,5 +7,13 @@ module Legitbot # :nodoc:
     domains 'google.com.', 'googlebot.com.'
   end
 
-  rule Legitbot::Google, %w[Googlebot Mediapartners-Google AdsBot-Google]
+  rule Legitbot::Google, %w[
+    APIs-Google
+    AdsBot-Google-Mobile
+    AdsBot-Google
+    Googlebot
+    Mediapartners-Google
+    AdsBot-Google-Mobile-Apps
+    FeedFetcher-Google
+  ]
 end

data/lib/legitbot/oracle.rb

@@ -3,7 +3,31 @@
 module Legitbot # :nodoc:
   # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
   class Oracle < BotMatch
-    ip_ranges '148.64.56.0/24'
+    ip_ranges %w[
+      132.145.9.5
+      132.145.11.125
+      132.145.14.70
+      132.145.15.209
+      132.145.64.33
+      132.145.66.116
+      132.145.66.156
+      132.145.67.248
+      140.238.81.78
+      140.238.83.181
+      140.238.94.137
+      140.238.95.47
+      140.238.95.199
+      152.67.128.219
+      152.67.137.35
+      152.67.138.180
+
+      148.64.56.64/28
+      148.64.56.79
+      148.64.56.80
+      148.64.56.112/28
+      148.64.56.127
+      148.64.56.128
+    ]
   end
 
   rule Legitbot::Oracle, %w[GrapeshotCrawler]

data/lib/legitbot/twitter.rb

@@ -7,6 +7,7 @@ module Legitbot # :nodoc:
     ip_ranges %w[
       199.16.156.0/22
       199.59.148.0/22
+      192.133.76.0/22
     ]
   end
 

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.4.2'
+  VERSION = '1.5.0'
 end

data/lib/legitbot.rb

@@ -5,6 +5,7 @@ require_relative 'legitbot/botmatch'
 
 require_relative 'legitbot/ahrefs'
 require_relative 'legitbot/alexa'
+require_relative 'legitbot/amazon'
 require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'

data/test/amazon_test.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class AmazonTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Amazon.new ip
+    assert !match.valid?, msg: "#{ip} is not a real AmazonAdBot IP"
+  end
+
+  def test_valid_ip
+    ip = '54.166.7.90'
+    match = Legitbot::Amazon.new ip
+    assert match.valid?, msg: "#{ip} is a valid AmazonAdBot IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'AmazonAdBot detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid AmazonAdBot'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+    assert bot, msg: 'AmazonAdBot detected from User-Agent'
+    assert bot.valid?, msg: 'Valid AmazonAdBot'
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+    assert_equal :amazon, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '81.1.172.108'
+    )
+    assert_equal :amazon, bot.detected_as
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-10 00:00:00.000000000 Z
+date: 2021-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: augmented_interval_tree
@@ -116,20 +116,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.22.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.22.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.22.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.22.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -150,6 +150,7 @@ files:
 - lib/legitbot.rb
 - lib/legitbot/ahrefs.rb
 - lib/legitbot/alexa.rb
+- lib/legitbot/amazon.rb
 - lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
@@ -169,6 +170,7 @@ files:
 - lib/legitbot/yandex.rb
 - test/ahrefs_test.rb
 - test/alexa_test.rb
+- test/amazon_test.rb
 - test/apple_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
@@ -214,6 +216,7 @@ test_files:
 - test/apple_test.rb
 - test/oracle_test.rb
 - test/google_test.rb
+- test/amazon_test.rb
 - test/petalbot_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb