legitbot 1.13.6 → 1.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -1
- data/README.md +1 -0
- data/lib/legitbot/anthropic.rb +109 -0
- data/lib/legitbot/duckduckgo.rb +5 -0
- data/lib/legitbot/legitbot.rb +1 -2
- data/lib/legitbot/openai.rb +5 -3
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot.rb +1 -0
- data/test/amazon_bot_test.rb +3 -6
- data/test/apple_test.rb +1 -2
- data/test/facebook_test.rb +1 -2
- data/test/legitbot/validators/domains_test.rb +1 -0
- data/test/legitbot/validators/ip_ranges_test.rb +2 -4
- data/test/lib/meta_ip_ranges_mock.rb +1 -2
- data/test/meta_test.rb +1 -2
- data/test/pinterest_test.rb +1 -2
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1354153ebd9a86c65859bcdcab109602ae84d73fb434bbe428dfff22af80a05d
|
|
4
|
+
data.tar.gz: 8b1fe76e0b50edffebbcec4d8c0994764c2b161fa93fb1e2812baed640a11f65
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '05997cc88feef7055eecc50db3c2544bb31f6dbae20d11b5a0f96c1e1b47e40128517ca6b07de45b5f8c96f5e229b7d993ea9804c4a1c7982603a3102e68fc61'
|
|
7
|
+
data.tar.gz: e1f4ae2b7a38a6653b79ce9490524df73070f0487b6766b304885af862574674e5a51737bd064695a195cfacfeb89fece4d64a7ed316c6330be106ee030799e4
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -52,6 +52,7 @@ end
|
|
|
52
52
|
- [Ahrefs](https://ahrefs.com/robot)
|
|
53
53
|
- [AmazonAdBot](https://adbot.amazon.com/)
|
|
54
54
|
- [AmazonBot](https://developer.amazon.com/amazonbot)
|
|
55
|
+
- [Anthropic Claude crawlers](https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)
|
|
55
56
|
- [Applebot](https://support.apple.com/en-us/119829)
|
|
56
57
|
- [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
|
|
57
58
|
- [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legitbot # :nodoc:
|
|
4
|
+
# https://support.claude.com/en/articles/8896518
|
|
5
|
+
class ClaudeBot < BotMatch
|
|
6
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
7
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
8
|
+
ip_ranges %w[
|
|
9
|
+
16.58.26.69/32
|
|
10
|
+
18.225.238.228/32
|
|
11
|
+
18.227.226.255/32
|
|
12
|
+
20.64.57.208/28
|
|
13
|
+
20.102.46.224/28
|
|
14
|
+
34.11.34.31/32
|
|
15
|
+
34.85.172.162/32
|
|
16
|
+
34.150.241.79/32
|
|
17
|
+
34.162.191.81/32
|
|
18
|
+
34.162.230.222/32
|
|
19
|
+
34.162.244.71/32
|
|
20
|
+
34.182.140.95/32
|
|
21
|
+
34.182.161.143/32
|
|
22
|
+
34.182.218.27/32
|
|
23
|
+
34.182.220.85/32
|
|
24
|
+
34.182.222.37/32
|
|
25
|
+
34.182.225.167/32
|
|
26
|
+
34.182.226.151/32
|
|
27
|
+
34.182.226.221/32
|
|
28
|
+
34.186.108.163/32
|
|
29
|
+
35.221.29.174/32
|
|
30
|
+
35.245.89.239/32
|
|
31
|
+
35.245.175.129/32
|
|
32
|
+
40.124.101.48/28
|
|
33
|
+
136.107.176.208/32
|
|
34
|
+
216.73.216.0/22
|
|
35
|
+
]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# https://support.claude.com/en/articles/8896518
|
|
39
|
+
class ClaudeUser < BotMatch
|
|
40
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
41
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
42
|
+
ip_ranges %w[
|
|
43
|
+
16.58.26.69/32
|
|
44
|
+
18.225.238.228/32
|
|
45
|
+
18.227.226.255/32
|
|
46
|
+
20.64.57.208/28
|
|
47
|
+
20.102.46.224/28
|
|
48
|
+
34.11.34.31/32
|
|
49
|
+
34.85.172.162/32
|
|
50
|
+
34.150.241.79/32
|
|
51
|
+
34.162.191.81/32
|
|
52
|
+
34.162.230.222/32
|
|
53
|
+
34.162.244.71/32
|
|
54
|
+
34.182.140.95/32
|
|
55
|
+
34.182.161.143/32
|
|
56
|
+
34.182.218.27/32
|
|
57
|
+
34.182.220.85/32
|
|
58
|
+
34.182.222.37/32
|
|
59
|
+
34.182.225.167/32
|
|
60
|
+
34.182.226.151/32
|
|
61
|
+
34.182.226.221/32
|
|
62
|
+
34.186.108.163/32
|
|
63
|
+
35.221.29.174/32
|
|
64
|
+
35.245.89.239/32
|
|
65
|
+
35.245.175.129/32
|
|
66
|
+
40.124.101.48/28
|
|
67
|
+
136.107.176.208/32
|
|
68
|
+
216.73.216.0/22
|
|
69
|
+
]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# https://support.claude.com/en/articles/8896518
|
|
73
|
+
class ClaudeSearchBot < BotMatch
|
|
74
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
75
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
76
|
+
ip_ranges %w[
|
|
77
|
+
16.58.26.69/32
|
|
78
|
+
18.225.238.228/32
|
|
79
|
+
18.227.226.255/32
|
|
80
|
+
20.64.57.208/28
|
|
81
|
+
20.102.46.224/28
|
|
82
|
+
34.11.34.31/32
|
|
83
|
+
34.85.172.162/32
|
|
84
|
+
34.150.241.79/32
|
|
85
|
+
34.162.191.81/32
|
|
86
|
+
34.162.230.222/32
|
|
87
|
+
34.162.244.71/32
|
|
88
|
+
34.182.140.95/32
|
|
89
|
+
34.182.161.143/32
|
|
90
|
+
34.182.218.27/32
|
|
91
|
+
34.182.220.85/32
|
|
92
|
+
34.182.222.37/32
|
|
93
|
+
34.182.225.167/32
|
|
94
|
+
34.182.226.151/32
|
|
95
|
+
34.182.226.221/32
|
|
96
|
+
34.186.108.163/32
|
|
97
|
+
35.221.29.174/32
|
|
98
|
+
35.245.89.239/32
|
|
99
|
+
35.245.175.129/32
|
|
100
|
+
40.124.101.48/28
|
|
101
|
+
136.107.176.208/32
|
|
102
|
+
216.73.216.0/22
|
|
103
|
+
]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
rule Legitbot::ClaudeBot, %w[ClaudeBot]
|
|
107
|
+
rule Legitbot::ClaudeUser, %w[Claude-User]
|
|
108
|
+
rule Legitbot::ClaudeSearchBot, %w[Claude-SearchBot]
|
|
109
|
+
end
|
data/lib/legitbot/duckduckgo.rb
CHANGED
|
@@ -7,7 +7,10 @@ module Legitbot # :nodoc:
|
|
|
7
7
|
# @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
|
|
8
8
|
# @fetch:selector main > article ul > li > div
|
|
9
9
|
ip_ranges %w[
|
|
10
|
+
4.144.148.23
|
|
10
11
|
4.144.182.50
|
|
12
|
+
4.144.227.142
|
|
13
|
+
4.144.251.39
|
|
11
14
|
4.150.142.218
|
|
12
15
|
4.156.30.123
|
|
13
16
|
4.156.136.79
|
|
@@ -326,6 +329,7 @@ module Legitbot # :nodoc:
|
|
|
326
329
|
132.164.209.198
|
|
327
330
|
132.220.1.71
|
|
328
331
|
134.33.207.189
|
|
332
|
+
135.171.251.169
|
|
329
333
|
135.234.221.112
|
|
330
334
|
172.168.43.43
|
|
331
335
|
172.168.53.53
|
|
@@ -336,6 +340,7 @@ module Legitbot # :nodoc:
|
|
|
336
340
|
172.169.28.184
|
|
337
341
|
172.169.60.134
|
|
338
342
|
172.169.177.131
|
|
343
|
+
172.188.250.220
|
|
339
344
|
172.189.40.18
|
|
340
345
|
172.193.197.146
|
|
341
346
|
172.193.245.229
|
data/lib/legitbot/legitbot.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Legitbot
|
|
|
18
18
|
# otherwise.
|
|
19
19
|
# :yields: a found bot
|
|
20
20
|
#
|
|
21
|
-
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
21
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
22
22
|
def self.bot(user_agent, ip)
|
|
23
23
|
bots = @rules
|
|
24
24
|
.select { |rule| rule[:fragments].any? { |f| user_agent&.index f } }
|
|
@@ -33,7 +33,6 @@ module Legitbot
|
|
|
33
33
|
selected
|
|
34
34
|
end
|
|
35
35
|
end
|
|
36
|
-
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
37
36
|
|
|
38
37
|
def self.rule(clazz, fragments)
|
|
39
38
|
@rules << { class: clazz, fragments: fragments }
|
data/lib/legitbot/openai.rb
CHANGED
|
@@ -31,7 +31,7 @@ module Legitbot # :nodoc:
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
# https://platform.openai.com/docs/bots
|
|
34
|
-
# rubocop:disable Metrics/ClassLength
|
|
34
|
+
# rubocop:disable-next Metrics/ClassLength
|
|
35
35
|
class OpenAIChat < BotMatch
|
|
36
36
|
# @fetch:url https://openai.com/chatgpt-user.json
|
|
37
37
|
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
@@ -177,7 +177,6 @@ module Legitbot # :nodoc:
|
|
|
177
177
|
52.242.132.240/28
|
|
178
178
|
52.255.109.80/28
|
|
179
179
|
52.255.109.96/28
|
|
180
|
-
52.255.109.112/28
|
|
181
180
|
52.255.109.144/28
|
|
182
181
|
52.255.111.0/28
|
|
183
182
|
52.255.111.80/28
|
|
@@ -238,11 +237,14 @@ module Legitbot # :nodoc:
|
|
|
238
237
|
172.207.1.32/28
|
|
239
238
|
172.212.172.160/28
|
|
240
239
|
172.215.215.32/28
|
|
240
|
+
191.232.238.96/28
|
|
241
|
+
191.233.197.0/28
|
|
241
242
|
191.233.199.160/28
|
|
243
|
+
191.234.167.144/28
|
|
244
|
+
191.235.99.160/28
|
|
242
245
|
191.237.249.64/28
|
|
243
246
|
]
|
|
244
247
|
end
|
|
245
|
-
# rubocop:enable Metrics/ClassLength
|
|
246
248
|
|
|
247
249
|
# https://platform.openai.com/docs/bots
|
|
248
250
|
class OpenAISearch < BotMatch
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot.rb
CHANGED
data/test/amazon_bot_test.rb
CHANGED
|
@@ -30,7 +30,7 @@ class AmazonBotTest < Minitest::Test
|
|
|
30
30
|
refute_predicate bot, :valid?
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
# rubocop:disable Layout/LineLength
|
|
33
|
+
# rubocop:disable-next Layout/LineLength
|
|
34
34
|
def test_user_agent
|
|
35
35
|
bot = Legitbot.bot(
|
|
36
36
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -40,9 +40,8 @@ class AmazonBotTest < Minitest::Test
|
|
|
40
40
|
assert bot
|
|
41
41
|
assert_predicate bot, :valid?
|
|
42
42
|
end
|
|
43
|
-
# rubocop:enable Layout/LineLength
|
|
44
43
|
|
|
45
|
-
# rubocop:disable Layout/LineLength
|
|
44
|
+
# rubocop:disable-next Layout/LineLength
|
|
46
45
|
def test_valid_name
|
|
47
46
|
bot = Legitbot.bot(
|
|
48
47
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -51,9 +50,8 @@ class AmazonBotTest < Minitest::Test
|
|
|
51
50
|
|
|
52
51
|
assert_equal :amazonbot, bot.detected_as
|
|
53
52
|
end
|
|
54
|
-
# rubocop:enable Layout/LineLength
|
|
55
53
|
|
|
56
|
-
# rubocop:disable Layout/LineLength
|
|
54
|
+
# rubocop:disable-next Layout/LineLength
|
|
57
55
|
def test_fake_name
|
|
58
56
|
bot = Legitbot.bot(
|
|
59
57
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -62,5 +60,4 @@ class AmazonBotTest < Minitest::Test
|
|
|
62
60
|
|
|
63
61
|
assert_equal :amazonbot, bot.detected_as
|
|
64
62
|
end
|
|
65
|
-
# rubocop:enable Layout/LineLength
|
|
66
63
|
end
|
data/test/apple_test.rb
CHANGED
|
@@ -20,7 +20,7 @@ class AppleTest < Minitest::Test
|
|
|
20
20
|
assert_predicate match, :fake?
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
# rubocop:disable Layout/LineLength
|
|
23
|
+
# rubocop:disable-next Layout/LineLength
|
|
24
24
|
def test_user_agent
|
|
25
25
|
bot = Legitbot.bot(
|
|
26
26
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
|
|
@@ -30,5 +30,4 @@ class AppleTest < Minitest::Test
|
|
|
30
30
|
assert_equal :apple, bot.detected_as
|
|
31
31
|
assert_predicate bot, :valid?
|
|
32
32
|
end
|
|
33
|
-
# rubocop:enable Layout/LineLength
|
|
34
33
|
end
|
data/test/facebook_test.rb
CHANGED
|
@@ -42,7 +42,7 @@ class FacebookTest < Minitest::Test
|
|
|
42
42
|
end
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
# rubocop:disable Layout/LineLength
|
|
45
|
+
# rubocop:disable-next Layout/LineLength
|
|
46
46
|
def test_user_agent3
|
|
47
47
|
Legitbot.bot(
|
|
48
48
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
|
|
@@ -52,5 +52,4 @@ class FacebookTest < Minitest::Test
|
|
|
52
52
|
assert_predicate bot, :fake?
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
|
-
# rubocop:enable Layout/LineLength
|
|
56
55
|
end
|
|
@@ -4,7 +4,7 @@ require 'minitest/autorun'
|
|
|
4
4
|
require 'legitbot'
|
|
5
5
|
|
|
6
6
|
module Legitbot
|
|
7
|
-
# rubocop:disable Minitest/MultipleAssertions
|
|
7
|
+
# rubocop:disable-next Minitest/MultipleAssertions
|
|
8
8
|
module Validators
|
|
9
9
|
class NoRanges
|
|
10
10
|
include IpRanges
|
|
@@ -113,7 +113,7 @@ module Legitbot
|
|
|
113
113
|
refute FlattenRanges.valid_ip?('2a03:2880:f233:0:0:0:0:1')
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
-
# rubocop:disable Metrics/AbcSize
|
|
116
|
+
# rubocop:disable-next Metrics/AbcSize
|
|
117
117
|
def test_matcher_loader
|
|
118
118
|
assert_equal 0, LoadRanges.counter
|
|
119
119
|
assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
|
@@ -128,7 +128,6 @@ module Legitbot
|
|
|
128
128
|
assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
|
129
129
|
assert_equal 2, LoadRanges.counter
|
|
130
130
|
end
|
|
131
|
-
# rubocop:enable Metrics/AbcSize
|
|
132
131
|
|
|
133
132
|
def test_nil_ranges
|
|
134
133
|
assert NilRanges.valid_ip?('127.0.0.1')
|
|
@@ -139,5 +138,4 @@ module Legitbot
|
|
|
139
138
|
end
|
|
140
139
|
end
|
|
141
140
|
end
|
|
142
|
-
# rubocop:enable Minitest/MultipleAssertions
|
|
143
141
|
end
|
|
@@ -4,10 +4,9 @@ module Legitbot
|
|
|
4
4
|
module MetaIpRanges
|
|
5
5
|
alias fetch_ip_ranges_orig fetch_ip_ranges
|
|
6
6
|
|
|
7
|
-
# rubocop:disable Layout/LineLength
|
|
7
|
+
# rubocop:disable-next Layout/LineLength
|
|
8
8
|
def fetch_ip_ranges
|
|
9
9
|
['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24']
|
|
10
10
|
end
|
|
11
|
-
# rubocop:enable Layout/LineLength
|
|
12
11
|
end
|
|
13
12
|
end
|
data/test/meta_test.rb
CHANGED
|
@@ -65,7 +65,7 @@ class MetaTest < Minitest::Test
|
|
|
65
65
|
end
|
|
66
66
|
end
|
|
67
67
|
|
|
68
|
-
# rubocop:disable Layout/LineLength
|
|
68
|
+
# rubocop:disable-next Layout/LineLength
|
|
69
69
|
def test_user_agent4
|
|
70
70
|
Legitbot.bot(
|
|
71
71
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 meta-externalagent/1.1 Twitterbot/1.0',
|
|
@@ -75,5 +75,4 @@ class MetaTest < Minitest::Test
|
|
|
75
75
|
assert_predicate bot, :fake?
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
|
-
# rubocop:enable Layout/LineLength
|
|
79
78
|
end
|
data/test/pinterest_test.rb
CHANGED
|
@@ -40,7 +40,7 @@ class PinterestTest < Minitest::Test
|
|
|
40
40
|
assert_predicate bot, :valid?
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
# rubocop:disable Layout/LineLength
|
|
43
|
+
# rubocop:disable-next Layout/LineLength
|
|
44
44
|
def test_android_not_bot
|
|
45
45
|
bot = Legitbot.bot(
|
|
46
46
|
'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]',
|
|
@@ -49,7 +49,6 @@ class PinterestTest < Minitest::Test
|
|
|
49
49
|
|
|
50
50
|
assert_nil bot
|
|
51
51
|
end
|
|
52
|
-
# rubocop:enable Layout/LineLength
|
|
53
52
|
|
|
54
53
|
def test_engine_name
|
|
55
54
|
bot = Legitbot.bot(
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legitbot
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.14.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexander Azarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-09-05 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fast_interval_tree
|
|
@@ -93,6 +93,7 @@ files:
|
|
|
93
93
|
- lib/legitbot.rb
|
|
94
94
|
- lib/legitbot/ahrefs.rb
|
|
95
95
|
- lib/legitbot/amazon.rb
|
|
96
|
+
- lib/legitbot/anthropic.rb
|
|
96
97
|
- lib/legitbot/apple.rb
|
|
97
98
|
- lib/legitbot/baidu.rb
|
|
98
99
|
- lib/legitbot/bing.rb
|