legitbot 1.11.12 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dae329e5de6473d5dc71904c8ca420808cb33e55dc90628846d032411d4d7132
-  data.tar.gz: 3f48ffc519567ea4c4612a42195425a4039dd03027e3a121dabec0b268ad14e4
+  metadata.gz: 8d61139628b7848376d114a6e077089f9c18f67e78187bdadd48333a06e6f866
+  data.tar.gz: 81025808fc543da79cbf0847b47b212cd2522ef4cfd3af648b7eb3c4bcd8e9a6
 SHA512:
-  metadata.gz: fcbe5171defe72ed8f1e5343296c7ca5b4a00909da05758369fe967972413c6d13fc70d18a194fd29a809554e1bba53309e63c2849cdb474803e9ced3474f183
-  data.tar.gz: 3eaf46e33803aed892f4409fbbd1318422f1d6c5aedba4e60899d005d194c1d3f93b95c740717ee2430b7a102c4085479729cf6da4b7efb9a2c693e9b09b95b6
+  metadata.gz: 8530f29add357c66339fd4d85aa96b338603347bc50d689dd3fa14e7e93913e76ff96460b6f21bf5757bb13f0a4e50c353e703518b83ffbf77cb514ee6f9b0a6
+  data.tar.gz: 63ec993dba0680788bc14a301cd8eaf9a07ba354d128bc8fc3c875312602fd441bafcd109621583f3d9e329c811851787c45d9dc46b828012f3001c26547492b

data/.github/workflows/autocorrect.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby: [3.1]
+        ruby: [3.2]
 
     steps:
       - uses: actions/checkout@v3

data/.github/workflows/pr.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [jruby, 3.0.6, 3.1, 3.2]
+        ruby: [jruby, 3.1, 3.2, 3.3, 3.4]
 
     steps:
       - uses: actions/checkout@v3

data/.ruby-version

@@ -1 +1 @@
-3.1.6
+3.2.7

data/README.md

@@ -50,14 +50,15 @@ end
 ## Supported
 
 - [Ahrefs](https://ahrefs.com/robot)
-- [Amazonbot](https://developer.amazon.com/amazonbot)
-- [Amazon AdBot](https://adbot.amazon.com/index.html)
+- [AmazonAdBot](https://adbot.amazon.com/)
+- [AmazonBot](https://developer.amazon.com/amazonbot)
 - [Applebot](https://support.apple.com/en-us/119829)
 - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
 - [BLEXBot (WebMeUp)](http://webmeup-crawler.com/)
 - [DataForSEO](https://dataforseo.com/dataforseo-bot)
-- [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
+- [DuckAssistBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot)
+- [DuckDuckBot](https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot)
 - [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 - [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
 - [OpenAI GPTBot](https://platform.openai.com/docs/gptbot)

data/legitbot.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
     'rubygems_mfa_required' => 'true'
   }
 
-  spec.required_ruby_version = '>= 3.0.0'
+  spec.required_ruby_version = '>= 3.1.0'
   spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
 

data/lib/legitbot/amazon.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
 module Legitbot # :nodoc:
-  # https://adbot.amazon.com/index.html
-  # https://developer.amazon.com/amazonbot
-  class Amazon < BotMatch
-    domains 'amazon.', 'amazonadbot.com.'
+  # https://adbot.amazon.com
+  class AmazonAdBot < BotMatch
+    domains 'amazonadbot.com.'
   end
 
-  rule Legitbot::Amazon, %w[Amazonbot AmazonAdBot]
+  # https://developer.amazon.com/en/amazonbot
+  class AmazonBot < BotMatch
+    domains 'crawl.amazonbot.amazon.'
+  end
+
+  rule Legitbot::AmazonBot, %w[Amazonbot]
+  rule Legitbot::AmazonAdBot, %w[AmazonAdBot]
 end

data/lib/legitbot/duckduckgo.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 module Legitbot # :nodoc:
-  # https://duckduckgo.com/duckduckbot
+  # https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
   # rubocop:disable Metrics/ClassLength
-  class DuckDuckGo < BotMatch
+  class DuckDuckBot < BotMatch
     # @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
     # @fetch:selector main > article ul > li > div
     ip_ranges %w[
+      4.156.136.79
       4.182.131.108
       4.195.133.120
       4.209.224.56
@@ -98,6 +99,7 @@ module Legitbot # :nodoc:
       20.219.45.67
       20.219.45.190
       20.226.133.105
+      20.232.184.230
       40.64.105.247
       40.64.106.11
       40.76.162.191
@@ -249,5 +251,256 @@ module Legitbot # :nodoc:
   end
   # rubocop:enable Metrics/ClassLength
 
-  rule Legitbot::DuckDuckGo, %w[DuckDuckBot]
+  # https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
+  # rubocop:disable Metrics/ClassLength
+  class DuckAssistBot < BotMatch
+    # @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot
+    # @fetch:selector main > article ul > li > div
+    ip_ranges %w[
+      4.156.136.79
+      4.182.131.108
+      4.195.133.120
+      4.209.224.56
+      4.213.46.14
+      4.228.76.163
+      13.89.106.77
+      20.3.1.178
+      20.12.141.99
+      20.40.133.240
+      20.43.150.85
+      20.43.150.93
+      20.43.172.120
+      20.44.222.1
+      20.49.136.28
+      20.50.48.159
+      20.50.48.192
+      20.50.49.0
+      20.50.49.25
+      20.50.49.40
+      20.50.49.55
+      20.50.49.237
+      20.50.50.46
+      20.50.50.118
+      20.50.50.121
+      20.50.50.123
+      20.50.50.130
+      20.50.50.134
+      20.50.50.145
+      20.50.50.146
+      20.50.50.163
+      20.50.168.91
+      20.53.78.106
+      20.53.78.123
+      20.53.78.138
+      20.53.78.144
+      20.53.78.236
+      20.53.91.2
+      20.53.92.211
+      20.53.134.160
+      20.56.197.58
+      20.56.197.63
+      20.61.34.40
+      20.62.224.44
+      20.71.12.143
+      20.72.242.93
+      20.73.132.240
+      20.73.202.147
+      20.75.144.152
+      20.79.226.26
+      20.79.238.198
+      20.79.239.66
+      20.80.129.80
+      20.93.28.24
+      20.99.255.235
+      20.113.3.121
+      20.113.14.159
+      20.185.79.15
+      20.185.79.47
+      20.191.44.16
+      20.191.44.22
+      20.191.44.119
+      20.191.44.234
+      20.191.45.212
+      20.193.12.126
+      20.193.24.10
+      20.193.24.251
+      20.193.25.197
+      20.193.27.215
+      20.193.45.113
+      20.195.108.47
+      20.197.209.11
+      20.197.209.27
+      20.201.15.208
+      20.204.240.172
+      20.204.241.148
+      20.204.242.19
+      20.204.242.101
+      20.204.243.55
+      20.204.246.81
+      20.204.246.254
+      20.207.72.11
+      20.207.72.21
+      20.207.72.110
+      20.207.72.113
+      20.207.97.190
+      20.207.99.197
+      20.207.107.181
+      20.219.43.246
+      20.219.45.67
+      20.219.45.190
+      20.226.133.105
+      20.232.184.230
+      40.64.105.247
+      40.64.106.11
+      40.76.162.191
+      40.76.162.208
+      40.76.162.247
+      40.76.163.7
+      40.76.163.23
+      40.76.173.151
+      40.80.242.63
+      40.81.250.205
+      40.88.21.235
+      40.89.243.175
+      40.114.182.45
+      40.114.182.153
+      40.114.182.172
+      40.114.183.88
+      40.114.183.196
+      40.114.183.251
+      40.119.232.50
+      40.119.232.146
+      40.119.232.215
+      40.119.232.218
+      40.119.232.251
+      51.8.71.117
+      51.8.253.152
+      51.104.146.225
+      51.104.146.235
+      51.104.160.167
+      51.104.160.177
+      51.104.161.32
+      51.104.162.149
+      51.104.163.250
+      51.104.164.109
+      51.104.164.147
+      51.104.164.189
+      51.104.164.215
+      51.104.166.111
+      51.104.167.19
+      51.104.167.52
+      51.104.167.54
+      51.104.167.61
+      51.104.167.71
+      51.104.167.87
+      51.104.167.88
+      51.104.167.95
+      51.104.167.96
+      51.104.167.104
+      51.104.167.110
+      51.104.180.26
+      51.104.180.47
+      51.104.180.53
+      51.107.40.209
+      51.116.131.221
+      51.120.48.122
+      51.138.90.161
+      51.138.90.206
+      51.138.90.233
+      52.142.24.149
+      52.142.26.175
+      52.143.95.162
+      52.143.95.204
+      52.143.241.111
+      52.143.242.6
+      52.143.243.117
+      52.143.244.81
+      52.143.247.235
+      52.146.58.236
+      52.146.59.12
+      52.146.59.154
+      52.146.59.156
+      52.146.63.80
+      52.148.161.87
+      52.148.165.38
+      52.149.25.43
+      52.149.28.18
+      52.149.28.83
+      52.149.30.45
+      52.149.56.151
+      52.149.58.27
+      52.149.58.69
+      52.149.58.139
+      52.149.58.173
+      52.149.60.38
+      52.149.61.51
+      52.154.60.82
+      52.154.169.50
+      52.154.169.200
+      52.154.170.26
+      52.154.170.28
+      52.154.170.88
+      52.154.170.96
+      52.154.170.113
+      52.154.170.117
+      52.154.170.122
+      52.154.170.209
+      52.154.170.229
+      52.154.170.243
+      52.154.171.0
+      52.154.171.44
+      52.154.171.70
+      52.154.171.87
+      52.154.171.150
+      52.154.171.196
+      52.154.171.205
+      52.154.171.235
+      52.154.171.250
+      52.154.172.2
+      52.190.37.160
+      52.224.16.221
+      52.224.16.229
+      52.224.19.152
+      52.224.20.174
+      52.224.20.181
+      52.224.20.186
+      52.224.20.190
+      52.224.20.193
+      52.224.20.203
+      52.224.20.204
+      52.224.20.223
+      52.224.20.227
+      52.224.20.249
+      52.224.21.4
+      52.224.21.19
+      52.224.21.20
+      52.224.21.23
+      52.224.21.27
+      52.224.21.49
+      52.224.21.51
+      52.224.21.53
+      52.224.21.55
+      52.224.21.61
+      52.242.224.168
+      57.152.72.128
+      104.43.54.127
+      104.43.55.116
+      104.43.55.117
+      104.43.55.166
+      104.43.55.167
+      108.141.83.74
+      172.169.17.165
+      191.233.3.197
+      191.233.3.202
+      191.234.216.4
+      191.234.216.178
+      191.235.201.214
+      191.235.202.38
+      191.235.202.48
+    ]
+  end
+  # rubocop:enable Metrics/ClassLength
+
+  rule Legitbot::DuckDuckBot, %w[DuckDuckBot]
+  rule Legitbot::DuckAssistBot, %w[DuckAssistBot]
 end

data/lib/legitbot/openai.rb

@@ -135,10 +135,26 @@ module Legitbot # :nodoc:
     # @fetch:url https://openai.com/searchbot.json
     # @fetch:jsonpath $.prefixes[*].ipv4Prefix
     ip_ranges %w[
+      20.14.99.96/28
+      20.25.151.224/28
       20.42.10.176/28
+      20.168.18.32/28
+      20.169.6.224/28
+      20.169.7.48/28
+      20.171.53.224/28
+      20.171.123.64/28
       51.8.102.0/24
       104.210.140.128/28
       135.234.64.0/24
+      172.182.193.80/28
+      172.182.193.224/28
+      172.182.194.32/28
+      172.182.194.144/28
+      172.182.195.48/28
+      172.182.209.208/28
+      172.182.211.192/28
+      172.182.213.192/28
+      172.182.224.0/28
       172.203.190.128/28
     ]
   end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.11.12'
+  VERSION = '1.12.0'
 end

data/test/amazon_ad_bot_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class AmazonAdBotTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::AmazonAdBot.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '54.166.7.90'
+    match = Legitbot::AmazonAdBot.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_user_agent
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '54.166.7.90'
+    )
+
+    assert_equal :amazonadbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      '81.1.172.108'
+    )
+
+    assert_equal :amazonadbot, bot.detected_as
+  end
+end

data/test/{amazon_test.rb → amazon_bot_test.rb}

@@ -2,20 +2,20 @@
 
 require_relative 'test_helper'
 
-class AmazonTest < Minitest::Test
+class AmazonBotTest < Minitest::Test
   include Minitest::Hooks
   include DnsServerMock
 
   def test_malicious_ip
     ip = '149.210.164.47'
-    match = Legitbot::Amazon.new ip
+    match = Legitbot::AmazonBot.new ip
 
     refute_predicate match, :valid?
   end
 
   def test_valid_ip
-    ip = '54.166.7.90'
-    match = Legitbot::Amazon.new ip
+    ip = '52.70.240.171'
+    match = Legitbot::AmazonBot.new ip
 
     assert_predicate match, :valid?
   end
@@ -30,18 +30,8 @@ class AmazonTest < Minitest::Test
     refute_predicate bot, :valid?
   end
 
-  def test_user_agent1
-    bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
-      '54.166.7.90'
-    )
-
-    assert bot
-    assert_predicate bot, :valid?
-  end
-
   # rubocop:disable Layout/LineLength
-  def test_user_agent2
+  def test_user_agent
     bot = Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '52.70.240.171'
@@ -52,32 +42,25 @@ class AmazonTest < Minitest::Test
   end
   # rubocop:enable Layout/LineLength
 
-  def test_valid_name1
-    bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
-      '54.166.7.90'
-    )
-
-    assert_equal :amazon, bot.detected_as
-  end
-
   # rubocop:disable Layout/LineLength
-  def test_valid_name2
+  def test_valid_name
     bot = Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '52.70.240.171'
     )
 
-    assert_equal :amazon, bot.detected_as
+    assert_equal :amazonbot, bot.detected_as
   end
   # rubocop:enable Layout/LineLength
 
+  # rubocop:disable Layout/LineLength
   def test_fake_name
     bot = Legitbot.bot(
-      'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
       '81.1.172.108'
     )
 
-    assert_equal :amazon, bot.detected_as
+    assert_equal :amazonbot, bot.detected_as
   end
+  # rubocop:enable Layout/LineLength
 end

data/test/lib/dns_server_mock.rb

@@ -22,13 +22,15 @@ TEST_DNS_RECORDS = {
     ptr: %w[crawl-52-86-176-3.alexa.com]
   },
 
-  # Amazon
+  # AmazonAdBot
   'crawler-54-166-7-90.amazonadbot.com' => {
     a: %w[54.166.7.90]
   },
   '54.166.7.90' => {
     ptr: %w[crawler-54-166-7-90.amazonadbot.com]
   },
+
+  # AmazonBot
   '52-70-240-171.crawl.amazonbot.amazon' => {
     a: %w[52.70.240.171]
   },

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.11.12
+  version: 1.12.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-08 00:00:00.000000000 Z
+date: 2025-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -103,7 +103,8 @@ files:
 - rakelib/console.rake
 - rakelib/test.rake
 - test/ahrefs_test.rb
-- test/amazon_test.rb
+- test/amazon_ad_bot_test.rb
+- test/amazon_bot_test.rb
 - test/apple_test.rb
 - test/blexbot_test.rb
 - test/botmatch_test.rb
@@ -136,14 +137,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Web crawler request validation