legitbot 1.4.3 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 395c6363b221d300574d3a0bd0d324f882589ca44afc8db86f9f39e42c61a44c
-  data.tar.gz: 58f7c5205cdc0e0cd138b00d244bca133c16424f1b16f7a99263c121d02ae5f9
+  metadata.gz: 90eab3a4d53cc388fac2db75058e11c549ec09b3acb3352a398c10e437c0f233
+  data.tar.gz: 5a252a30d37a44de60bb5b7baf1f264477211c22c0acc1477317f8281b7e9103
 SHA512:
-  metadata.gz: df1f576284899afa5386cdea651a1cc811f9ddbf6cb683fdf85af71a829fedeae91995858e2453ed4743bd86105b3796a87cc2269f420e144f59faa5f982b646
-  data.tar.gz: 7f06269a4c24d08f957e47551a85338986e004f817a2d68ddacbc0354919fdcaef2106478ed2fce32133665343628b0b8ba922d0958485ee20ca98808ca3dbff
+  metadata.gz: c56a32b2bccb2667f5c42c7580c31b2d79884331dfecf5f3cd01f1242ad2e53a5a4d1192a01f1f814014ef3da38702ad5b6041da3ba89d6b14cfbb5658550c47
+  data.tar.gz: c388aad0dec6a86a8d7a9ae23ae1e3200cd7eb3df421f74934b1a7582d9e63c4da1b66d9f28def1685f821c9ac2813b2d7fa38c4653a0a72301576c1cdddcf9b

data/.rubocop.yml

@@ -8,3 +8,6 @@ Gemspec/RequiredRubyVersion:
 
 Naming/MemoizedInstanceVariableName:
   Enabled: false
+
+Style/MapToHash:
+  Enabled: false

data/README.md

@@ -39,10 +39,18 @@ Rack::Attack.blocklist 'fake search engines' do |request|
 end
 ```
 
+## Versioning
+
+[Semantic versioning](https://semver.org/) with the following clarifications:
+
+* MINOR version is incremented when support for new bots is added.
+* PATCH version is incremented when validation logic for a bot changes (IP list updated, for example).
+
 ## Supported
 
 * [Ahrefs](https://ahrefs.com/robot)
 * [Alexa](https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-)
+* [Amazon AdBot](https://adbot.amazon.com/index.html)
 * [Applebot](https://support.apple.com/en-us/HT204683)
 * [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 * [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
@@ -59,7 +67,7 @@ end
 
 Apache 2.0
 
-## References
+## Other projects
 
 * Play Framework variant in Scala: [play-legitbot](https://github.com/osinka/play-legitbot)
 * Article [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
@@ -72,3 +80,5 @@ Apache 2.0
   classify IP as a search engine, but also label them as suspicious and
   reports the number of days since the last activity. My implementation of
   the protocol in Scala is [here](https://github.com/osinka/httpbl).
+* [CIDRAM](https://github.com/CIDRAM/CIDRAM) is a PHP routing manager with built-in support
+  to validate bots.

data/legitbot.gemspec

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'English'
+
 $LOAD_PATH.push File.expand_path('lib', __dir__)
 require 'legitbot/version'
 
@@ -13,6 +15,9 @@ Gem::Specification.new do |spec|
   spec.homepage = 'https://github.com/alaz/legitbot'
   spec.summary = 'Validate requests from Web crawlers: impersonating or not?'
   spec.description = 'Does Web request come from a real search engine or from an impersonating agent?'
+  spec.metadata = {
+    'rubygems_mfa_required' => 'true'
+  }
 
   spec.required_ruby_version = '>= 2.5.0'
   spec.add_dependency 'augmented_interval_tree', '~> 0.1', '>= 0.1.1'
@@ -20,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
   spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
   spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.20.0', '>= 1.20.0'
+  spec.add_development_dependency 'rubocop', '~> 1.24.0', '>= 1.24.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/amazon.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://adbot.amazon.com/index.html
+  class Amazon < BotMatch
+    domains 'amazonadbot.com.'
+  end
+
+  rule Legitbot::Amazon, %w[AmazonAdBot]
+end

data/lib/legitbot/duckduckgo.rb

@@ -5,16 +5,17 @@ module Legitbot # :nodoc:
   class DuckDuckGo < BotMatch
     ip_ranges %w[
       20.191.45.212
-      23.21.227.69
       40.88.21.235
-      50.16.241.113
-      50.16.241.114
-      50.16.241.117
-      50.16.247.234
-      52.5.190.19
-      52.204.97.54
-      54.197.234.188
-      54.208.100.253
+      40.76.173.151
+      40.76.163.7
+      20.185.79.47
+      52.142.26.175
+      20.185.79.15
+      52.142.24.149
+      40.76.162.208
+      40.76.163.23
+      40.76.162.191
+      40.76.162.247
       54.208.102.37
       107.21.1.8
     ]

data/lib/legitbot/facebook.rb

@@ -18,5 +18,8 @@ module Legitbot # :nodoc:
     end
   end
 
-  rule Legitbot::Facebook, %w[Facebot facebookexternalhit/1.1]
+  rule Legitbot::Facebook, %w[
+    facebookexternalhit/1.1
+    facebookcatalog/1.0
+  ]
 end

data/lib/legitbot/google.rb

@@ -7,5 +7,13 @@ module Legitbot # :nodoc:
     domains 'google.com.', 'googlebot.com.'
   end
 
-  rule Legitbot::Google, %w[Googlebot Mediapartners-Google AdsBot-Google]
+  rule Legitbot::Google, %w[
+    APIs-Google
+    AdsBot-Google-Mobile
+    AdsBot-Google
+    Googlebot
+    Mediapartners-Google
+    AdsBot-Google-Mobile-Apps
+    FeedFetcher-Google
+  ]
 end

data/lib/legitbot/twitter.rb

@@ -7,6 +7,7 @@ module Legitbot # :nodoc:
     ip_ranges %w[
       199.16.156.0/22
       199.59.148.0/22
+      192.133.76.0/22
     ]
   end
 

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.4.3'
+  VERSION = '1.5.1'
 end

data/lib/legitbot.rb

@@ -5,6 +5,7 @@ require_relative 'legitbot/botmatch'
 
 require_relative 'legitbot/ahrefs'
 require_relative 'legitbot/alexa'
+require_relative 'legitbot/amazon'
 require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'

data/test/amazon_test.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class AmazonTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Amazon.new ip
+    assert !match.valid?, msg: "#{ip} is not a real AmazonAdBot IP"
+  end
+
+  def test_valid_ip
+    ip = '54.166.7.90'
+    match = Legitbot::Amazon.new ip
+    assert match.valid?, msg: "#{ip} is a valid AmazonAdBot IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'AmazonAdBot detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid AmazonAdBot'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+    assert bot, msg: 'AmazonAdBot detected from User-Agent'
+    assert bot.valid?, msg: 'Valid AmazonAdBot'
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+    assert_equal :amazon, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '81.1.172.108'
+    )
+    assert_equal :amazon, bot.detected_as
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.1
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-10 00:00:00.000000000 Z
+date: 2022-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: augmented_interval_tree
@@ -116,20 +116,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.24.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.24.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.24.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.24.0
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -150,6 +150,7 @@ files:
 - lib/legitbot.rb
 - lib/legitbot/ahrefs.rb
 - lib/legitbot/alexa.rb
+- lib/legitbot/amazon.rb
 - lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
@@ -169,6 +170,7 @@ files:
 - lib/legitbot/yandex.rb
 - test/ahrefs_test.rb
 - test/alexa_test.rb
+- test/amazon_test.rb
 - test/apple_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
@@ -183,7 +185,8 @@ files:
 homepage: https://github.com/alaz/legitbot
 licenses:
 - Apache-2.0
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options:
 - "--charset=UTF-8"
@@ -214,6 +217,7 @@ test_files:
 - test/apple_test.rb
 - test/oracle_test.rb
 - test/google_test.rb
+- test/amazon_test.rb
 - test/petalbot_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb