legitbot 1.11.13 → 1.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4e488aedddc99f554b6c77583c9c83a002e619d67abd39be7a5c6e7ffa91648
-  data.tar.gz: 42d72ad19facb323cef74b12557e7ff11ef296670c3792e966f1696232266214
+  metadata.gz: d56f4dd32414017581a905af1e021634e30d22c4e76c4146c7e3a50923277a02
+  data.tar.gz: 47fe62aa1baf95dee8a3e3cee3e815ec28aa4dc13ec077318b18b67214a6dc5d
 SHA512:
-  metadata.gz: 39f5610ea3341cc4f8ac6e8e9c1a89b22393880f89f841b7ede8e1a2091fede2bd80a0bf8ca944887a3ebc20b6283ee497f46f2cfa8a28fe50ed111d391a35ea
-  data.tar.gz: 07ecd3ffaab28b65e8d6d2a424cd534b197d6c42595342afc4a5268cd946ca221989f45cec4c16984e9b0b8de52554d56a5e4f5c71478fdda4bb5802d2b0318c
+  metadata.gz: 2fd237294ef2d18e7c23dcd91c11e0b397d0db982fce65b4e3d2429a0da2dc26e8707b9d4c2aa8f6eac4ffb2c508c70364ea39007551ee6386292e20d32a72f2
+  data.tar.gz: 4bc856536c0187d2bcebef7959f59becb69398685aa1917fb978bd2888649707e4072dc039a7aca8f88561e75f79686f8307538c733913ba1ca9d0157294728b

data/.github/workflows/autocorrect.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby: [3.1]
+        ruby: [3.2]
 
     steps:
       - uses: actions/checkout@v3

data/.github/workflows/pr.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [jruby, 3.0.6, 3.1, 3.2]
+        ruby: [jruby, 3.1, 3.2, 3.3, 3.4]
 
     steps:
       - uses: actions/checkout@v3

data/.ruby-version

@@ -1 +1 @@
-3.1.6
+3.2.7

data/README.md

@@ -50,14 +50,15 @@ end
 ## Supported
 
 - [Ahrefs](https://ahrefs.com/robot)
-- [Amazonbot](https://developer.amazon.com/amazonbot)
-- [Amazon AdBot](https://adbot.amazon.com/index.html)
+- [AmazonAdBot](https://adbot.amazon.com/)
+- [AmazonBot](https://developer.amazon.com/amazonbot)
 - [Applebot](https://support.apple.com/en-us/119829)
 - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
 - [BLEXBot (WebMeUp)](http://webmeup-crawler.com/)
 - [DataForSEO](https://dataforseo.com/dataforseo-bot)
-- [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
+- [DuckAssistBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot)
+- [DuckDuckBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot)
 - [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 - [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
 - [OpenAI GPTBot](https://platform.openai.com/docs/gptbot)

data/legitbot.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
     'rubygems_mfa_required' => 'true'
   }
 
-  spec.required_ruby_version = '>= 3.0.0'
+  spec.required_ruby_version = '>= 3.1.0'
   spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
 

data/lib/legitbot/amazon.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
 module Legitbot # :nodoc:
-  # https://adbot.amazon.com/index.html
-  # https://developer.amazon.com/amazonbot
-  class Amazon < BotMatch
-    domains 'amazon.', 'amazonadbot.com.'
+  # https://adbot.amazon.com
+  class AmazonAdBot < BotMatch
+    domains 'amazonadbot.com.'
   end
 
-  rule Legitbot::Amazon, %w[Amazonbot AmazonAdBot]
+  # https://developer.amazon.com/en/amazonbot
+  class AmazonBot < BotMatch
+    domains 'crawl.amazonbot.amazon.'
+  end
+
+  rule Legitbot::AmazonBot, %w[Amazonbot]
+  rule Legitbot::AmazonAdBot, %w[AmazonAdBot]
 end

data/lib/legitbot/duckduckgo.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 module Legitbot # :nodoc:
-  # https://duckduckgo.com/duckduckbot
+  # https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
   # rubocop:disable Metrics/ClassLength
-  class DuckDuckGo < BotMatch
+  class DuckDuckBot < BotMatch
     # @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
     # @fetch:selector main > article ul > li > div
     ip_ranges %w[
@@ -251,5 +251,256 @@ module Legitbot # :nodoc:
   end
   # rubocop:enable Metrics/ClassLength
 
-  rule Legitbot::DuckDuckGo, %w[DuckDuckBot]
+  # https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
+  # rubocop:disable Metrics/ClassLength
+  class DuckAssistBot < BotMatch
+    # @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
+    # @fetch:selector main > article ul > li > div
+    ip_ranges %w[
+      4.156.136.79
+      4.182.131.108
+      4.195.133.120
+      4.209.224.56
+      4.213.46.14
+      4.228.76.163
+      13.89.106.77
+      20.3.1.178
+      20.12.141.99
+      20.40.133.240
+      20.43.150.85
+      20.43.150.93
+      20.43.172.120
+      20.44.222.1
+      20.49.136.28
+      20.50.48.159
+      20.50.48.192
+      20.50.49.0
+      20.50.49.25
+      20.50.49.40
+      20.50.49.55
+      20.50.49.237
+      20.50.50.46
+      20.50.50.118
+      20.50.50.121
+      20.50.50.123
+      20.50.50.130
+      20.50.50.134
+      20.50.50.145
+      20.50.50.146
+      20.50.50.163
+      20.50.168.91
+      20.53.78.106
+      20.53.78.123
+      20.53.78.138
+      20.53.78.144
+      20.53.78.236
+      20.53.91.2
+      20.53.92.211
+      20.53.134.160
+      20.56.197.58
+      20.56.197.63
+      20.61.34.40
+      20.62.224.44
+      20.71.12.143
+      20.72.242.93
+      20.73.132.240
+      20.73.202.147
+      20.75.144.152
+      20.79.226.26
+      20.79.238.198
+      20.79.239.66
+      20.80.129.80
+      20.93.28.24
+      20.99.255.235
+      20.113.3.121
+      20.113.14.159
+      20.185.79.15
+      20.185.79.47
+      20.191.44.16
+      20.191.44.22
+      20.191.44.119
+      20.191.44.234
+      20.191.45.212
+      20.193.12.126
+      20.193.24.10
+      20.193.24.251
+      20.193.25.197
+      20.193.27.215
+      20.193.45.113
+      20.195.108.47
+      20.197.209.11
+      20.197.209.27
+      20.201.15.208
+      20.204.240.172
+      20.204.241.148
+      20.204.242.19
+      20.204.242.101
+      20.204.243.55
+      20.204.246.81
+      20.204.246.254
+      20.207.72.11
+      20.207.72.21
+      20.207.72.110
+      20.207.72.113
+      20.207.97.190
+      20.207.99.197
+      20.207.107.181
+      20.219.43.246
+      20.219.45.67
+      20.219.45.190
+      20.226.133.105
+      20.232.184.230
+      40.64.105.247
+      40.64.106.11
+      40.76.162.191
+      40.76.162.208
+      40.76.162.247
+      40.76.163.7
+      40.76.163.23
+      40.76.173.151
+      40.80.242.63
+      40.81.250.205
+      40.88.21.235
+      40.89.243.175
+      40.114.182.45
+      40.114.182.153
+      40.114.182.172
+      40.114.183.88
+      40.114.183.196
+      40.114.183.251
+      40.119.232.50
+      40.119.232.146
+      40.119.232.215
+      40.119.232.218
+      40.119.232.251
+      51.8.71.117
+      51.8.253.152
+      51.104.146.225
+      51.104.146.235
+      51.104.160.167
+      51.104.160.177
+      51.104.161.32
+      51.104.162.149
+      51.104.163.250
+      51.104.164.109
+      51.104.164.147
+      51.104.164.189
+      51.104.164.215
+      51.104.166.111
+      51.104.167.19
+      51.104.167.52
+      51.104.167.54
+      51.104.167.61
+      51.104.167.71
+      51.104.167.87
+      51.104.167.88
+      51.104.167.95
+      51.104.167.96
+      51.104.167.104
+      51.104.167.110
+      51.104.180.26
+      51.104.180.47
+      51.104.180.53
+      51.107.40.209
+      51.116.131.221
+      51.120.48.122
+      51.138.90.161
+      51.138.90.206
+      51.138.90.233
+      52.142.24.149
+      52.142.26.175
+      52.143.95.162
+      52.143.95.204
+      52.143.241.111
+      52.143.242.6
+      52.143.243.117
+      52.143.244.81
+      52.143.247.235
+      52.146.58.236
+      52.146.59.12
+      52.146.59.154
+      52.146.59.156
+      52.146.63.80
+      52.148.161.87
+      52.148.165.38
+      52.149.25.43
+      52.149.28.18
+      52.149.28.83
+      52.149.30.45
+      52.149.56.151
+      52.149.58.27
+      52.149.58.69
+      52.149.58.139
+      52.149.58.173
+      52.149.60.38
+      52.149.61.51
+      52.154.60.82
+      52.154.169.50
+      52.154.169.200
+      52.154.170.26
+      52.154.170.28
+      52.154.170.88
+      52.154.170.96
+      52.154.170.113
+      52.154.170.117
+      52.154.170.122
+      52.154.170.209
+      52.154.170.229
+      52.154.170.243
+      52.154.171.0
+      52.154.171.44
+      52.154.171.70
+      52.154.171.87
+      52.154.171.150
+      52.154.171.196
+      52.154.171.205
+      52.154.171.235
+      52.154.171.250
+      52.154.172.2
+      52.190.37.160
+      52.224.16.221
+      52.224.16.229
+      52.224.19.152
+      52.224.20.174
+      52.224.20.181
+      52.224.20.186
+      52.224.20.190
+      52.224.20.193
+      52.224.20.203
+      52.224.20.204
+      52.224.20.223
+      52.224.20.227
+      52.224.20.249
+      52.224.21.4
+      52.224.21.19
+      52.224.21.20
+      52.224.21.23
+      52.224.21.27
+      52.224.21.49
+      52.224.21.51
+      52.224.21.53
+      52.224.21.55
+      52.224.21.61
+      52.242.224.168
+      57.152.72.128
+      104.43.54.127
+      104.43.55.116
+      104.43.55.117
+      104.43.55.166
+      104.43.55.167
+      108.141.83.74
+      172.169.17.165
+      191.233.3.197
+      191.233.3.202
+      191.234.216.4
+      191.234.216.178
+      191.235.201.214
+      191.235.202.38
+      191.235.202.48
+    ]
+  end
+  # rubocop:enable Metrics/ClassLength
+
+  rule Legitbot::DuckDuckBot, %w[DuckDuckBot]
+  rule Legitbot::DuckAssistBot, %w[DuckAssistBot]
 end

data/lib/legitbot/openai.rb

@@ -27,46 +27,70 @@ module Legitbot # :nodoc:
       4.151.119.48/28
       4.151.241.240/28
       4.196.118.112/28
+      4.196.198.80/28
+      4.197.19.176/28
       4.197.22.112/28
       4.197.115.112/28
       4.205.128.176/28
+      4.234.83.96/28
       13.65.138.96/28
       13.65.138.112/28
+      13.70.107.160/28
+      13.76.32.208/28
       13.76.116.80/28
       13.83.167.128/28
       13.83.237.176/28
+      20.0.53.96/28
       20.55.229.144/28
       20.63.221.64/28
+      20.77.178.240/28
       20.90.7.144/28
       20.97.189.96/28
       20.102.212.144/28
       20.117.22.224/28
       20.161.75.208/28
       20.193.50.32/28
+      20.194.0.208/28
+      20.194.1.0/28
+      20.194.157.176/28
+      20.210.154.128/28
       20.215.188.192/28
       20.215.214.16/28
       20.228.106.176/28
+      20.235.75.208/28
       20.235.87.224/28
       20.249.63.208/28
+      23.97.109.224/28
+      23.98.142.176/28
       23.98.179.16/28
       23.98.186.64/28
       23.98.186.96/28
       23.98.186.176/28
       23.98.186.192/28
+      23.102.140.144/28
+      23.102.141.32/28
+      40.75.14.224/28
       40.84.181.32/28
       40.84.221.208/28
       40.84.221.224/28
       40.116.73.208/28
       51.8.155.48/28
       51.8.155.64/28
+      51.8.155.80/28
       51.8.155.112/28
       52.148.129.32/28
+      52.154.22.48/28
       52.156.77.144/28
       52.159.227.32/28
       52.159.249.96/28
+      52.173.123.0/28
+      52.173.219.96/28
+      52.173.219.112/28
       52.173.234.16/28
       52.173.234.80/28
+      52.173.235.80/28
       52.176.139.176/28
+      52.187.246.128/28
       52.190.137.16/28
       52.190.137.144/28
       52.190.139.48/28
@@ -78,6 +102,7 @@ module Legitbot # :nodoc:
       52.236.94.144/28
       52.242.132.224/28
       52.242.132.240/28
+      52.242.245.208/28
       52.252.113.240/28
       52.255.109.80/28
       52.255.109.96/28
@@ -94,6 +119,7 @@ module Legitbot # :nodoc:
       57.154.175.0/28
       57.154.187.32/28
       68.154.28.96/28
+      68.218.30.112/28
       68.220.57.64/28
       68.221.67.160/28
       68.221.67.192/28
@@ -115,17 +141,27 @@ module Legitbot # :nodoc:
       135.237.133.48/28
       135.237.133.112/28
       137.135.191.176/28
+      138.91.30.48/28
+      138.91.46.96/28
       172.178.140.144/28
       172.178.141.112/28
       172.178.141.128/28
       172.183.143.224/28
       172.183.222.128/28
+      172.204.16.64/28
       172.212.159.64/28
       172.213.11.144/28
       172.213.12.112/28
       172.213.21.16/28
       172.213.21.112/28
       172.213.21.144/28
+      191.233.194.32/28
+      191.233.196.112/28
+      191.234.167.128/28
+      191.235.66.16/28
+      191.235.98.144/28
+      191.235.99.80/28
+      191.239.245.16/28
     ]
   end
   # rubocop:enable Metrics/ClassLength

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.11.13'
+  VERSION = '1.12.1'
 end

data/test/amazon_ad_bot_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class AmazonAdBotTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::AmazonAdBot.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '54.166.7.90'
+    match = Legitbot::AmazonAdBot.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_user_agent
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+
+    assert_equal :amazonadbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '81.1.172.108'
+    )
+
+    assert_equal :amazonadbot, bot.detected_as
+  end
+end

data/test/{amazon_test.rb → amazon_bot_test.rb}

@@ -2,20 +2,20 @@
 
 require_relative 'test_helper'
 
-class AmazonTest < Minitest::Test
+class AmazonBotTest < Minitest::Test
   include Minitest::Hooks
   include DnsServerMock
 
   def test_malicious_ip
     ip = '149.210.164.47'
-    match = Legitbot::Amazon.new ip
+    match = Legitbot::AmazonBot.new ip
 
     refute_predicate match, :valid?
   end
 
   def test_valid_ip
-    ip = '54.166.7.90'
-    match = Legitbot::Amazon.new ip
+    ip = '52.70.240.171'
+    match = Legitbot::AmazonBot.new ip
 
     assert_predicate match, :valid?
   end
@@ -30,18 +30,8 @@ class AmazonTest < Minitest::Test
     refute_predicate bot, :valid?
   end
 
-  def test_user_agent1
-    bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
-      '54.166.7.90'
-    )
-
-    assert bot
-    assert_predicate bot, :valid?
-  end
-
   # rubocop:disable Layout/LineLength
-  def test_user_agent2
+  def test_user_agent
     bot = Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '52.70.240.171'
@@ -52,32 +42,25 @@ class AmazonTest < Minitest::Test
   end
   # rubocop:enable Layout/LineLength
 
-  def test_valid_name1
-    bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
-      '54.166.7.90'
-    )
-
-    assert_equal :amazon, bot.detected_as
-  end
-
   # rubocop:disable Layout/LineLength
-  def test_valid_name2
+  def test_valid_name
     bot = Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '52.70.240.171'
     )
 
-    assert_equal :amazon, bot.detected_as
+    assert_equal :amazonbot, bot.detected_as
   end
   # rubocop:enable Layout/LineLength
 
+  # rubocop:disable Layout/LineLength
   def test_fake_name
     bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '81.1.172.108'
     )
 
-    assert_equal :amazon, bot.detected_as
+    assert_equal :amazonbot, bot.detected_as
   end
+  # rubocop:enable Layout/LineLength
 end

data/test/legitbot/validators/domains_test.rb

@@ -11,6 +11,7 @@ module Legitbot
 
     class DomainMatch
       include Domains
+
       domains 'search.msn.com', reverse: false
 
       @resolver = Minitest::Mock.new
@@ -27,6 +28,7 @@ module Legitbot
 
     class ReverseMatch
       include Domains
+
       domains 'search.msn.com'
     end
 

data/test/legitbot/validators/ip_ranges_test.rb

@@ -12,16 +12,19 @@ module Legitbot
 
     class ArrayRanges
       include IpRanges
+
       ip_ranges '66.220.144.0/21', '2a03:2880:f234::/48'
     end
 
     class FlattenRanges
       include IpRanges
+
       ip_ranges %w[66.220.144.0/21 2a03:2880:f234::/48]
     end
 
     class EmptyRanges
       include IpRanges
+
       ip_ranges
 
       def initialize(ip)
@@ -49,11 +52,13 @@ module Legitbot
 
     class NilRanges
       include IpRanges
+
       ip_ranges { nil }
     end
 
     class Ipv4Ranges
       include IpRanges
+
       ip_ranges { ['66.220.144.0/21'] }
     end
 

data/test/lib/dns_server_mock.rb

@@ -22,13 +22,15 @@ TEST_DNS_RECORDS = {
     ptr: %w[crawl-52-86-176-3.alexa.com]
   },
 
-  # Amazon
+  # AmazonAdBot
   'crawler-54-166-7-90.amazonadbot.com' => {
     a: %w[54.166.7.90]
   },
   '54.166.7.90' => {
     ptr: %w[crawler-54-166-7-90.amazonadbot.com]
   },
+
+  # AmazonBot
   '52-70-240-171.crawl.amazonbot.amazon' => {
     a: %w[52.70.240.171]
   },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.11.13
+  version: 1.12.1
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-14 00:00:00.000000000 Z
+date: 2025-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -103,7 +103,8 @@ files:
 - rakelib/console.rake
 - rakelib/test.rake
 - test/ahrefs_test.rb
-- test/amazon_test.rb
+- test/amazon_ad_bot_test.rb
+- test/amazon_bot_test.rb
 - test/apple_test.rb
 - test/blexbot_test.rb
 - test/botmatch_test.rb
@@ -136,14 +137,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Web crawler request validation