legitbot 1.10.6 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6dbba3fc1b7fd156d5d560786ac18939387e2e059ce7f3385a105adf1c10ee0d
-  data.tar.gz: b030a92a6210016021debd9df09db177366a90eb7f841037e8f3b093319e161c
+  metadata.gz: ccf22fe1fab3a7cab2955709eb3b0ada75a66305b255a36ec795eb092d9741c8
+  data.tar.gz: 6b03643eb517f59626c7a1e59b04f6808ce067b87597bf5f7e8486f4c1eb309c
 SHA512:
-  metadata.gz: 9451f510ddc37f1cb57be2143a496189c3422fb7754b92ddcd5a2acdf8896873093933ebc4b427948f2c452abeb610027f1dc557dcf0f8687e662cd75f1d4be2
-  data.tar.gz: 45e258d414f71c5df1e9642a8549defdce012ab96bbf6d133a34a3003620dbb03d14ca74cc75e5d96dd50759b634535e343446823fbffb0ffc77ce28b458b442
+  metadata.gz: 44f09102368337c185aa95c32a76dd551a01bf9fda2e098757383101bb3c57cced58f85fcad00f7c86dda1550df142b0a9549ea2d7c9c54c41cb2fffe7bfbabe
+  data.tar.gz: 6149c2e4eca68be2224ee529184a4df492fd224602aed1c08aa0a80af5b77ac902e23258331099c769ea4cb80f07c4b99df00526626d328fa13852afedb57eb0

data/README.md

@@ -50,10 +50,13 @@ end
 ## Supported
 
 - [Ahrefs](https://ahrefs.com/robot)
+- [Amazonbot](https://developer.amazon.com/amazonbot)
 - [Amazon AdBot](https://adbot.amazon.com/index.html)
 - [Applebot](https://support.apple.com/en-us/119829)
 - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
+- [BLEXBot (WebMeUp)](http://webmeup-crawler.com/)
+- [DataForSEO](https://dataforseo.com/dataforseo-bot)
 - [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
 - [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 - [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)

data/lib/legitbot/amazon.rb

@@ -2,9 +2,10 @@
 
 module Legitbot # :nodoc:
   # https://adbot.amazon.com/index.html
+  # https://developer.amazon.com/amazonbot
   class Amazon < BotMatch
-    domains 'amazonadbot.com.'
+    domains 'amazon.', 'amazonadbot.com.'
   end
 
-  rule Legitbot::Amazon, %w[AmazonAdBot]
+  rule Legitbot::Amazon, %w[Amazonbot AmazonAdBot]
 end

data/lib/legitbot/blexbot.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # http://webmeup-crawler.com/
+  class BLEXBot < BotMatch
+    domains 'webmeup.com.'
+  end
+
+  rule Legitbot::BLEXBot, %w[BLEXBot]
+end

data/lib/legitbot/dataforseo.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://dataforseo.com/dataforseo-bot
+  class DataForSEO < BotMatch
+    domains 'dataforseo.com.'
+  end
+
+  rule Legitbot::DataForSEO, %w[DataForSeoBot]
+end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.10.6'
+  VERSION = '1.11.0'
 end

data/lib/legitbot.rb

@@ -8,6 +8,8 @@ require_relative 'legitbot/amazon'
 require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'
+require_relative 'legitbot/blexbot'
+require_relative 'legitbot/dataforseo'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'

data/test/amazon_test.rb

@@ -30,7 +30,7 @@ class AmazonTest < Minitest::Test
     refute_predicate bot, :valid?
   end
 
-  def test_valid_ua
+  def test_user_agent1
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '54.166.7.90'
@@ -40,7 +40,19 @@ class AmazonTest < Minitest::Test
     assert_predicate bot, :valid?
   end
 
-  def test_valid_name
+  # rubocop:disable Layout/LineLength
+  def test_user_agent2
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
+      '52.70.240.171'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+  # rubocop:enable Layout/LineLength
+
+  def test_valid_name1
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '54.166.7.90'
@@ -49,6 +61,17 @@ class AmazonTest < Minitest::Test
     assert_equal :amazon, bot.detected_as
   end
 
+  # rubocop:disable Layout/LineLength
+  def test_valid_name2
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
+      '52.70.240.171'
+    )
+
+    assert_equal :amazon, bot.detected_as
+  end
+  # rubocop:enable Layout/LineLength
+
   def test_fake_name
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',

data/test/blexbot_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class BLEXBot < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::BLEXBot.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '65.21.113.197'
+    match = Legitbot::BLEXBot.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '65.21.113.197'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '65.21.113.197'
+    )
+
+    assert_equal :blexbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '81.1.172.108'
+    )
+
+    assert_equal :blexbot, bot.detected_as
+  end
+end

data/test/dataforseo_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class DataForSEOTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::DataForSEO.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '136.243.228.176'
+    match = Legitbot::DataForSEO.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '136.243.228.176'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '136.243.228.176'
+    )
+
+    assert_equal :dataforseo, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '81.1.172.108'
+    )
+
+    assert_equal :dataforseo, bot.detected_as
+  end
+end

data/test/lib/dns_server_mock.rb

@@ -29,6 +29,12 @@ TEST_DNS_RECORDS = {
   '54.166.7.90' => {
     ptr: %w[crawler-54-166-7-90.amazonadbot.com]
   },
+  '52-70-240-171.crawl.amazonbot.amazon' => {
+    a: %w[52.70.240.171]
+  },
+  '52.70.240.171' => {
+    ptr: %w[52-70-240-171.crawl.amazonbot.amazon]
+  },
 
   # Apple
   '17-58-98-60.applebot.apple.com' => {
@@ -38,6 +44,21 @@ TEST_DNS_RECORDS = {
     ptr: %w[17-58-98-60.applebot.apple.com]
   },
 
+  # BLEXBot (WebMeUp)
+  'pot22.webmeup.com' => {
+    a: %w[65.21.113.197]
+  },
+  '65.21.113.197' => {
+    ptr: %w[pot22.webmeup.com]
+  },
+  # DataForSEO
+  'crawling-gateway-136-243-228-176.dataforseo.com' => {
+    a: %w[136.243.228.176]
+  },
+  '136.243.228.176' => {
+    ptr: %w[crawling-gateway-136-243-228-176.dataforseo.com]
+  },
+
   # Google
   'crawl-66-249-64-141.googlebot.com' => {
     a: %w[66.249.64.141]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.10.6
+  version: 1.11.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-09 00:00:00.000000000 Z
+date: 2024-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -76,8 +76,10 @@ files:
 - lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
+- lib/legitbot/blexbot.rb
 - lib/legitbot/botmatch.rb
 - lib/legitbot/config/resolver.rb
+- lib/legitbot/dataforseo.rb
 - lib/legitbot/duckduckgo.rb
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
@@ -104,7 +106,9 @@ files:
 - test/ahrefs_test.rb
 - test/amazon_test.rb
 - test/apple_test.rb
+- test/blexbot_test.rb
 - test/botmatch_test.rb
+- test/dataforseo_test.rb
 - test/facebook_test.rb
 - test/google_test.rb
 - test/ias_test.rb