legitbot 1.13.6 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a1e3df9f4373997b0fcd35aa98643c5a166d8497bd4771f23d203a60fcd50db2
4
- data.tar.gz: e9e4802223d647f4d644fbde0324d8c579691fa898622a0c63f5a7e7c2d8267a
3
+ metadata.gz: fa929afd499bc170d126d73acb0f79e7e9ed7bb463acb159a9f414e252a449cd
4
+ data.tar.gz: 8c98051ad4853574f31773e902b3c975ad73358c8274960c4e5627557f226646
5
5
  SHA512:
6
- metadata.gz: 66510e6126601bedc266e2db46ad1497af8e0efce3a6cd6cc60758797b41f7a47e3195d83b0a883c4697263a24aab3cc8f0091ef76825df5fdf711aec700570c
7
- data.tar.gz: 077f7c66fbeaef6c8e37c9b3f456c170d9dc6349ad570edfd88a114e2403cbd83c0260cced58715a73bf520e34f9343f8f714d80b2b8eaeae5c476fe99e9fac2
6
+ metadata.gz: cd2e7cb65efb4c7723d1a6c72258695880de61c6dd0b39620b0df80d51aa1af34d9d935f087f72a4b418fc63dd1d34f0080ee04d5d9cf59f95be2a9cb93e3b57
7
+ data.tar.gz: 227a52ad9e1f72efc9ce7921ea95cc90deca8512b7fd5f196f82ee4971b83a6796ddf8ee5e3a6591e4a04371aa2fad1f7bd2c58da923f143eff35aa03121b0e8
data/README.md CHANGED
@@ -52,6 +52,7 @@ end
52
52
  - [Ahrefs](https://ahrefs.com/robot)
53
53
  - [AmazonAdBot](https://adbot.amazon.com/)
54
54
  - [AmazonBot](https://developer.amazon.com/amazonbot)
55
+ - [Anthropic Claude crawlers](https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)
55
56
  - [Applebot](https://support.apple.com/en-us/119829)
56
57
  - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
57
58
  - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Legitbot # :nodoc:
4
+ # https://support.claude.com/en/articles/8896518
5
+ class ClaudeBot < BotMatch
6
+ # @fetch:url https://claude.com/crawling/bots.json
7
+ # @fetch:jsonpath $.prefixes[*].ipv4Prefix
8
+ ip_ranges %w[
9
+ 16.58.26.69/32
10
+ 18.225.238.228/32
11
+ 18.227.226.255/32
12
+ 20.64.57.208/28
13
+ 20.102.46.224/28
14
+ 34.11.34.31/32
15
+ 34.85.172.162/32
16
+ 34.150.241.79/32
17
+ 34.162.191.81/32
18
+ 34.162.230.222/32
19
+ 34.162.244.71/32
20
+ 34.182.140.95/32
21
+ 34.182.161.143/32
22
+ 34.182.218.27/32
23
+ 34.182.220.85/32
24
+ 34.182.222.37/32
25
+ 34.182.225.167/32
26
+ 34.182.226.151/32
27
+ 34.182.226.221/32
28
+ 34.186.108.163/32
29
+ 35.221.29.174/32
30
+ 35.245.89.239/32
31
+ 35.245.175.129/32
32
+ 40.124.101.48/28
33
+ 136.107.176.208/32
34
+ 216.73.216.0/22
35
+ ]
36
+ end
37
+
38
+ # https://support.claude.com/en/articles/8896518
39
+ class ClaudeUser < BotMatch
40
+ # @fetch:url https://claude.com/crawling/bots.json
41
+ # @fetch:jsonpath $.prefixes[*].ipv4Prefix
42
+ ip_ranges %w[
43
+ 16.58.26.69/32
44
+ 18.225.238.228/32
45
+ 18.227.226.255/32
46
+ 20.64.57.208/28
47
+ 20.102.46.224/28
48
+ 34.11.34.31/32
49
+ 34.85.172.162/32
50
+ 34.150.241.79/32
51
+ 34.162.191.81/32
52
+ 34.162.230.222/32
53
+ 34.162.244.71/32
54
+ 34.182.140.95/32
55
+ 34.182.161.143/32
56
+ 34.182.218.27/32
57
+ 34.182.220.85/32
58
+ 34.182.222.37/32
59
+ 34.182.225.167/32
60
+ 34.182.226.151/32
61
+ 34.182.226.221/32
62
+ 34.186.108.163/32
63
+ 35.221.29.174/32
64
+ 35.245.89.239/32
65
+ 35.245.175.129/32
66
+ 40.124.101.48/28
67
+ 136.107.176.208/32
68
+ 216.73.216.0/22
69
+ ]
70
+ end
71
+
72
+ # https://support.claude.com/en/articles/8896518
73
+ class ClaudeSearchBot < BotMatch
74
+ # @fetch:url https://claude.com/crawling/bots.json
75
+ # @fetch:jsonpath $.prefixes[*].ipv4Prefix
76
+ ip_ranges %w[
77
+ 16.58.26.69/32
78
+ 18.225.238.228/32
79
+ 18.227.226.255/32
80
+ 20.64.57.208/28
81
+ 20.102.46.224/28
82
+ 34.11.34.31/32
83
+ 34.85.172.162/32
84
+ 34.150.241.79/32
85
+ 34.162.191.81/32
86
+ 34.162.230.222/32
87
+ 34.162.244.71/32
88
+ 34.182.140.95/32
89
+ 34.182.161.143/32
90
+ 34.182.218.27/32
91
+ 34.182.220.85/32
92
+ 34.182.222.37/32
93
+ 34.182.225.167/32
94
+ 34.182.226.151/32
95
+ 34.182.226.221/32
96
+ 34.186.108.163/32
97
+ 35.221.29.174/32
98
+ 35.245.89.239/32
99
+ 35.245.175.129/32
100
+ 40.124.101.48/28
101
+ 136.107.176.208/32
102
+ 216.73.216.0/22
103
+ ]
104
+ end
105
+
106
+ rule Legitbot::ClaudeBot, %w[ClaudeBot]
107
+ rule Legitbot::ClaudeUser, %w[Claude-User]
108
+ rule Legitbot::ClaudeSearchBot, %w[Claude-SearchBot]
109
+ end
@@ -336,6 +336,7 @@ module Legitbot # :nodoc:
336
336
  172.169.28.184
337
337
  172.169.60.134
338
338
  172.169.177.131
339
+ 172.188.250.220
339
340
  172.189.40.18
340
341
  172.193.197.146
341
342
  172.193.245.229
@@ -18,7 +18,7 @@ module Legitbot
18
18
  # otherwise.
19
19
  # :yields: a found bot
20
20
  #
21
- # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
21
+ # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
22
22
  def self.bot(user_agent, ip)
23
23
  bots = @rules
24
24
  .select { |rule| rule[:fragments].any? { |f| user_agent&.index f } }
@@ -33,7 +33,6 @@ module Legitbot
33
33
  selected
34
34
  end
35
35
  end
36
- # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
37
36
 
38
37
  def self.rule(clazz, fragments)
39
38
  @rules << { class: clazz, fragments: fragments }
@@ -31,7 +31,7 @@ module Legitbot # :nodoc:
31
31
  end
32
32
 
33
33
  # https://platform.openai.com/docs/bots
34
- # rubocop:disable Metrics/ClassLength
34
+ # rubocop:disable-next Metrics/ClassLength
35
35
  class OpenAIChat < BotMatch
36
36
  # @fetch:url https://openai.com/chatgpt-user.json
37
37
  # @fetch:jsonpath $.prefixes[*].ipv4Prefix
@@ -242,7 +242,6 @@ module Legitbot # :nodoc:
242
242
  191.237.249.64/28
243
243
  ]
244
244
  end
245
- # rubocop:enable Metrics/ClassLength
246
245
 
247
246
  # https://platform.openai.com/docs/bots
248
247
  class OpenAISearch < BotMatch
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Legitbot
4
- VERSION = '1.13.6'
4
+ VERSION = '1.14.0'
5
5
  end
data/lib/legitbot.rb CHANGED
@@ -5,6 +5,7 @@ require_relative 'legitbot/botmatch'
5
5
 
6
6
  require_relative 'legitbot/ahrefs'
7
7
  require_relative 'legitbot/amazon'
8
+ require_relative 'legitbot/anthropic'
8
9
  require_relative 'legitbot/apple'
9
10
  require_relative 'legitbot/baidu'
10
11
  require_relative 'legitbot/bing'
@@ -30,7 +30,7 @@ class AmazonBotTest < Minitest::Test
30
30
  refute_predicate bot, :valid?
31
31
  end
32
32
 
33
- # rubocop:disable Layout/LineLength
33
+ # rubocop:disable-next Layout/LineLength
34
34
  def test_user_agent
35
35
  bot = Legitbot.bot(
36
36
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
@@ -40,9 +40,8 @@ class AmazonBotTest < Minitest::Test
40
40
  assert bot
41
41
  assert_predicate bot, :valid?
42
42
  end
43
- # rubocop:enable Layout/LineLength
44
43
 
45
- # rubocop:disable Layout/LineLength
44
+ # rubocop:disable-next Layout/LineLength
46
45
  def test_valid_name
47
46
  bot = Legitbot.bot(
48
47
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
@@ -51,9 +50,8 @@ class AmazonBotTest < Minitest::Test
51
50
 
52
51
  assert_equal :amazonbot, bot.detected_as
53
52
  end
54
- # rubocop:enable Layout/LineLength
55
53
 
56
- # rubocop:disable Layout/LineLength
54
+ # rubocop:disable-next Layout/LineLength
57
55
  def test_fake_name
58
56
  bot = Legitbot.bot(
59
57
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
@@ -62,5 +60,4 @@ class AmazonBotTest < Minitest::Test
62
60
 
63
61
  assert_equal :amazonbot, bot.detected_as
64
62
  end
65
- # rubocop:enable Layout/LineLength
66
63
  end
data/test/apple_test.rb CHANGED
@@ -20,7 +20,7 @@ class AppleTest < Minitest::Test
20
20
  assert_predicate match, :fake?
21
21
  end
22
22
 
23
- # rubocop:disable Layout/LineLength
23
+ # rubocop:disable-next Layout/LineLength
24
24
  def test_user_agent
25
25
  bot = Legitbot.bot(
26
26
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
@@ -30,5 +30,4 @@ class AppleTest < Minitest::Test
30
30
  assert_equal :apple, bot.detected_as
31
31
  assert_predicate bot, :valid?
32
32
  end
33
- # rubocop:enable Layout/LineLength
34
33
  end
@@ -42,7 +42,7 @@ class FacebookTest < Minitest::Test
42
42
  end
43
43
  end
44
44
 
45
- # rubocop:disable Layout/LineLength
45
+ # rubocop:disable-next Layout/LineLength
46
46
  def test_user_agent3
47
47
  Legitbot.bot(
48
48
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
@@ -52,5 +52,4 @@ class FacebookTest < Minitest::Test
52
52
  assert_predicate bot, :fake?
53
53
  end
54
54
  end
55
- # rubocop:enable Layout/LineLength
56
55
  end
@@ -4,7 +4,7 @@ require 'minitest/autorun'
4
4
  require 'legitbot'
5
5
 
6
6
  module Legitbot
7
- # rubocop:disable Minitest/MultipleAssertions
7
+ # rubocop:disable-next Minitest/MultipleAssertions
8
8
  module Validators
9
9
  class NoRanges
10
10
  include IpRanges
@@ -113,7 +113,7 @@ module Legitbot
113
113
  refute FlattenRanges.valid_ip?('2a03:2880:f233:0:0:0:0:1')
114
114
  end
115
115
 
116
- # rubocop:disable Metrics/AbcSize
116
+ # rubocop:disable-next Metrics/AbcSize
117
117
  def test_matcher_loader
118
118
  assert_equal 0, LoadRanges.counter
119
119
  assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
@@ -128,7 +128,6 @@ module Legitbot
128
128
  assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
129
129
  assert_equal 2, LoadRanges.counter
130
130
  end
131
- # rubocop:enable Metrics/AbcSize
132
131
 
133
132
  def test_nil_ranges
134
133
  assert NilRanges.valid_ip?('127.0.0.1')
@@ -139,5 +138,4 @@ module Legitbot
139
138
  end
140
139
  end
141
140
  end
142
- # rubocop:enable Minitest/MultipleAssertions
143
141
  end
@@ -4,10 +4,9 @@ module Legitbot
4
4
  module MetaIpRanges
5
5
  alias fetch_ip_ranges_orig fetch_ip_ranges
6
6
 
7
- # rubocop:disable Layout/LineLength
7
+ # rubocop:disable-next Layout/LineLength
8
8
  def fetch_ip_ranges
9
9
  ['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24']
10
10
  end
11
- # rubocop:enable Layout/LineLength
12
11
  end
13
12
  end
data/test/meta_test.rb CHANGED
@@ -65,7 +65,7 @@ class MetaTest < Minitest::Test
65
65
  end
66
66
  end
67
67
 
68
- # rubocop:disable Layout/LineLength
68
+ # rubocop:disable-next Layout/LineLength
69
69
  def test_user_agent4
70
70
  Legitbot.bot(
71
71
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 meta-externalagent/1.1 Twitterbot/1.0',
@@ -75,5 +75,4 @@ class MetaTest < Minitest::Test
75
75
  assert_predicate bot, :fake?
76
76
  end
77
77
  end
78
- # rubocop:enable Layout/LineLength
79
78
  end
@@ -40,7 +40,7 @@ class PinterestTest < Minitest::Test
40
40
  assert_predicate bot, :valid?
41
41
  end
42
42
 
43
- # rubocop:disable Layout/LineLength
43
+ # rubocop:disable-next Layout/LineLength
44
44
  def test_android_not_bot
45
45
  bot = Legitbot.bot(
46
46
  'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]',
@@ -49,7 +49,6 @@ class PinterestTest < Minitest::Test
49
49
 
50
50
  assert_nil bot
51
51
  end
52
- # rubocop:enable Layout/LineLength
53
52
 
54
53
  def test_engine_name
55
54
  bot = Legitbot.bot(
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: legitbot
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.13.6
4
+ version: 1.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexander Azarov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2026-08-22 00:00:00.000000000 Z
11
+ date: 2026-08-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fast_interval_tree
@@ -93,6 +93,7 @@ files:
93
93
  - lib/legitbot.rb
94
94
  - lib/legitbot/ahrefs.rb
95
95
  - lib/legitbot/amazon.rb
96
+ - lib/legitbot/anthropic.rb
96
97
  - lib/legitbot/apple.rb
97
98
  - lib/legitbot/baidu.rb
98
99
  - lib/legitbot/bing.rb