legitbot 1.13.5 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/legitbot/ahrefs.rb +30 -0
- data/lib/legitbot/anthropic.rb +109 -0
- data/lib/legitbot/duckduckgo.rb +1 -0
- data/lib/legitbot/legitbot.rb +1 -2
- data/lib/legitbot/openai.rb +1 -56
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot.rb +1 -0
- data/test/amazon_bot_test.rb +3 -6
- data/test/apple_test.rb +1 -2
- data/test/facebook_test.rb +1 -2
- data/test/legitbot/validators/ip_ranges_test.rb +2 -4
- data/test/lib/meta_ip_ranges_mock.rb +1 -2
- data/test/meta_test.rb +1 -2
- data/test/pinterest_test.rb +1 -2
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fa929afd499bc170d126d73acb0f79e7e9ed7bb463acb159a9f414e252a449cd
|
|
4
|
+
data.tar.gz: 8c98051ad4853574f31773e902b3c975ad73358c8274960c4e5627557f226646
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cd2e7cb65efb4c7723d1a6c72258695880de61c6dd0b39620b0df80d51aa1af34d9d935f087f72a4b418fc63dd1d34f0080ee04d5d9cf59f95be2a9cb93e3b57
|
|
7
|
+
data.tar.gz: 227a52ad9e1f72efc9ce7921ea95cc90deca8512b7fd5f196f82ee4971b83a6796ddf8ee5e3a6591e4a04371aa2fad1f7bd2c58da923f143eff35aa03121b0e8
|
data/README.md
CHANGED
|
@@ -52,6 +52,7 @@ end
|
|
|
52
52
|
- [Ahrefs](https://ahrefs.com/robot)
|
|
53
53
|
- [AmazonAdBot](https://adbot.amazon.com/)
|
|
54
54
|
- [AmazonBot](https://developer.amazon.com/amazonbot)
|
|
55
|
+
- [Anthropic Claude crawlers](https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)
|
|
55
56
|
- [Applebot](https://support.apple.com/en-us/119829)
|
|
56
57
|
- [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
|
|
57
58
|
- [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
|
data/lib/legitbot/ahrefs.rb
CHANGED
|
@@ -14,6 +14,36 @@ module Legitbot # :nodoc:
|
|
|
14
14
|
37.59.204.128/27
|
|
15
15
|
51.68.247.192/27
|
|
16
16
|
51.75.236.128/27
|
|
17
|
+
51.81.103.240/28
|
|
18
|
+
51.81.104.32/28
|
|
19
|
+
51.81.105.16/28
|
|
20
|
+
51.81.110.16/28
|
|
21
|
+
51.81.110.48/28
|
|
22
|
+
51.81.110.80/28
|
|
23
|
+
51.81.111.0/28
|
|
24
|
+
51.81.111.64/28
|
|
25
|
+
51.81.111.112/28
|
|
26
|
+
51.81.111.128/28
|
|
27
|
+
51.81.163.16/28
|
|
28
|
+
51.81.163.48/28
|
|
29
|
+
51.81.163.64/28
|
|
30
|
+
51.81.164.144/28
|
|
31
|
+
51.81.164.192/28
|
|
32
|
+
51.81.165.80/28
|
|
33
|
+
51.81.165.208/28
|
|
34
|
+
51.81.169.112/28
|
|
35
|
+
51.81.169.192/28
|
|
36
|
+
51.81.170.0/28
|
|
37
|
+
51.89.38.128/28
|
|
38
|
+
51.89.39.48/28
|
|
39
|
+
51.89.59.32/28
|
|
40
|
+
51.89.69.32/28
|
|
41
|
+
51.89.69.96/28
|
|
42
|
+
51.89.71.80/28
|
|
43
|
+
51.89.71.144/28
|
|
44
|
+
51.89.72.0/28
|
|
45
|
+
51.89.74.192/28
|
|
46
|
+
51.89.83.48/28
|
|
17
47
|
51.89.129.0/24
|
|
18
48
|
51.161.37.0/24
|
|
19
49
|
51.161.65.0/24
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legitbot # :nodoc:
|
|
4
|
+
# https://support.claude.com/en/articles/8896518
|
|
5
|
+
class ClaudeBot < BotMatch
|
|
6
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
7
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
8
|
+
ip_ranges %w[
|
|
9
|
+
16.58.26.69/32
|
|
10
|
+
18.225.238.228/32
|
|
11
|
+
18.227.226.255/32
|
|
12
|
+
20.64.57.208/28
|
|
13
|
+
20.102.46.224/28
|
|
14
|
+
34.11.34.31/32
|
|
15
|
+
34.85.172.162/32
|
|
16
|
+
34.150.241.79/32
|
|
17
|
+
34.162.191.81/32
|
|
18
|
+
34.162.230.222/32
|
|
19
|
+
34.162.244.71/32
|
|
20
|
+
34.182.140.95/32
|
|
21
|
+
34.182.161.143/32
|
|
22
|
+
34.182.218.27/32
|
|
23
|
+
34.182.220.85/32
|
|
24
|
+
34.182.222.37/32
|
|
25
|
+
34.182.225.167/32
|
|
26
|
+
34.182.226.151/32
|
|
27
|
+
34.182.226.221/32
|
|
28
|
+
34.186.108.163/32
|
|
29
|
+
35.221.29.174/32
|
|
30
|
+
35.245.89.239/32
|
|
31
|
+
35.245.175.129/32
|
|
32
|
+
40.124.101.48/28
|
|
33
|
+
136.107.176.208/32
|
|
34
|
+
216.73.216.0/22
|
|
35
|
+
]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# https://support.claude.com/en/articles/8896518
|
|
39
|
+
class ClaudeUser < BotMatch
|
|
40
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
41
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
42
|
+
ip_ranges %w[
|
|
43
|
+
16.58.26.69/32
|
|
44
|
+
18.225.238.228/32
|
|
45
|
+
18.227.226.255/32
|
|
46
|
+
20.64.57.208/28
|
|
47
|
+
20.102.46.224/28
|
|
48
|
+
34.11.34.31/32
|
|
49
|
+
34.85.172.162/32
|
|
50
|
+
34.150.241.79/32
|
|
51
|
+
34.162.191.81/32
|
|
52
|
+
34.162.230.222/32
|
|
53
|
+
34.162.244.71/32
|
|
54
|
+
34.182.140.95/32
|
|
55
|
+
34.182.161.143/32
|
|
56
|
+
34.182.218.27/32
|
|
57
|
+
34.182.220.85/32
|
|
58
|
+
34.182.222.37/32
|
|
59
|
+
34.182.225.167/32
|
|
60
|
+
34.182.226.151/32
|
|
61
|
+
34.182.226.221/32
|
|
62
|
+
34.186.108.163/32
|
|
63
|
+
35.221.29.174/32
|
|
64
|
+
35.245.89.239/32
|
|
65
|
+
35.245.175.129/32
|
|
66
|
+
40.124.101.48/28
|
|
67
|
+
136.107.176.208/32
|
|
68
|
+
216.73.216.0/22
|
|
69
|
+
]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# https://support.claude.com/en/articles/8896518
|
|
73
|
+
class ClaudeSearchBot < BotMatch
|
|
74
|
+
# @fetch:url https://claude.com/crawling/bots.json
|
|
75
|
+
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
76
|
+
ip_ranges %w[
|
|
77
|
+
16.58.26.69/32
|
|
78
|
+
18.225.238.228/32
|
|
79
|
+
18.227.226.255/32
|
|
80
|
+
20.64.57.208/28
|
|
81
|
+
20.102.46.224/28
|
|
82
|
+
34.11.34.31/32
|
|
83
|
+
34.85.172.162/32
|
|
84
|
+
34.150.241.79/32
|
|
85
|
+
34.162.191.81/32
|
|
86
|
+
34.162.230.222/32
|
|
87
|
+
34.162.244.71/32
|
|
88
|
+
34.182.140.95/32
|
|
89
|
+
34.182.161.143/32
|
|
90
|
+
34.182.218.27/32
|
|
91
|
+
34.182.220.85/32
|
|
92
|
+
34.182.222.37/32
|
|
93
|
+
34.182.225.167/32
|
|
94
|
+
34.182.226.151/32
|
|
95
|
+
34.182.226.221/32
|
|
96
|
+
34.186.108.163/32
|
|
97
|
+
35.221.29.174/32
|
|
98
|
+
35.245.89.239/32
|
|
99
|
+
35.245.175.129/32
|
|
100
|
+
40.124.101.48/28
|
|
101
|
+
136.107.176.208/32
|
|
102
|
+
216.73.216.0/22
|
|
103
|
+
]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
rule Legitbot::ClaudeBot, %w[ClaudeBot]
|
|
107
|
+
rule Legitbot::ClaudeUser, %w[Claude-User]
|
|
108
|
+
rule Legitbot::ClaudeSearchBot, %w[Claude-SearchBot]
|
|
109
|
+
end
|
data/lib/legitbot/duckduckgo.rb
CHANGED
data/lib/legitbot/legitbot.rb
CHANGED
|
@@ -18,7 +18,7 @@ module Legitbot
|
|
|
18
18
|
# otherwise.
|
|
19
19
|
# :yields: a found bot
|
|
20
20
|
#
|
|
21
|
-
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
21
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
22
22
|
def self.bot(user_agent, ip)
|
|
23
23
|
bots = @rules
|
|
24
24
|
.select { |rule| rule[:fragments].any? { |f| user_agent&.index f } }
|
|
@@ -33,7 +33,6 @@ module Legitbot
|
|
|
33
33
|
selected
|
|
34
34
|
end
|
|
35
35
|
end
|
|
36
|
-
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
37
36
|
|
|
38
37
|
def self.rule(clazz, fragments)
|
|
39
38
|
@rules << { class: clazz, fragments: fragments }
|
data/lib/legitbot/openai.rb
CHANGED
|
@@ -31,27 +31,22 @@ module Legitbot # :nodoc:
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
# https://platform.openai.com/docs/bots
|
|
34
|
-
# rubocop:disable Metrics/ClassLength
|
|
34
|
+
# rubocop:disable-next Metrics/ClassLength
|
|
35
35
|
class OpenAIChat < BotMatch
|
|
36
36
|
# @fetch:url https://openai.com/chatgpt-user.json
|
|
37
37
|
# @fetch:jsonpath $.prefixes[*].ipv4Prefix
|
|
38
38
|
ip_ranges %w[
|
|
39
39
|
4.151.71.176/28
|
|
40
40
|
4.151.119.48/28
|
|
41
|
-
4.151.241.240/28
|
|
42
41
|
4.189.118.208/28
|
|
43
42
|
4.189.119.48/28
|
|
44
|
-
4.197.19.176/28
|
|
45
43
|
4.197.22.112/28
|
|
46
44
|
4.197.64.0/28
|
|
47
45
|
4.197.64.16/28
|
|
48
46
|
4.197.64.48/28
|
|
49
47
|
4.197.64.64/28
|
|
50
48
|
4.197.115.112/28
|
|
51
|
-
4.198.72.16/28
|
|
52
|
-
4.198.96.112/28
|
|
53
49
|
4.201.232.64/28
|
|
54
|
-
4.201.232.80/28
|
|
55
50
|
4.205.128.176/28
|
|
56
51
|
4.218.24.64/28
|
|
57
52
|
4.226.200.16/28
|
|
@@ -75,7 +70,6 @@ module Legitbot # :nodoc:
|
|
|
75
70
|
9.223.181.208/28
|
|
76
71
|
9.234.96.192/28
|
|
77
72
|
9.234.97.96/28
|
|
78
|
-
9.234.97.128/28
|
|
79
73
|
13.65.138.112/28
|
|
80
74
|
13.67.72.16/28
|
|
81
75
|
13.71.2.208/28
|
|
@@ -83,8 +77,6 @@ module Legitbot # :nodoc:
|
|
|
83
77
|
13.76.116.80/28
|
|
84
78
|
13.83.167.128/28
|
|
85
79
|
13.83.237.176/28
|
|
86
|
-
20.17.108.96/28
|
|
87
|
-
20.27.94.128/28
|
|
88
80
|
20.45.178.144/28
|
|
89
81
|
20.55.229.144/28
|
|
90
82
|
20.57.199.192/28
|
|
@@ -98,16 +90,12 @@ module Legitbot # :nodoc:
|
|
|
98
90
|
20.125.144.144/28
|
|
99
91
|
20.161.75.208/28
|
|
100
92
|
20.168.7.192/28
|
|
101
|
-
20.168.7.240/28
|
|
102
93
|
20.169.73.32/28
|
|
103
94
|
20.169.73.64/28
|
|
104
95
|
20.169.78.48/28
|
|
105
96
|
20.169.78.64/28
|
|
106
|
-
20.169.78.96/28
|
|
107
|
-
20.169.78.112/28
|
|
108
97
|
20.169.78.128/28
|
|
109
98
|
20.169.78.160/28
|
|
110
|
-
20.169.78.176/28
|
|
111
99
|
20.169.78.192/28
|
|
112
100
|
20.169.86.224/28
|
|
113
101
|
20.169.86.240/28
|
|
@@ -117,27 +105,17 @@ module Legitbot # :nodoc:
|
|
|
117
105
|
20.170.184.64/28
|
|
118
106
|
20.170.184.80/28
|
|
119
107
|
20.172.29.32/28
|
|
120
|
-
20.193.233.240/28
|
|
121
|
-
20.194.0.208/28
|
|
122
|
-
20.194.1.0/28
|
|
123
108
|
20.199.211.160/28
|
|
124
109
|
20.199.242.0/28
|
|
125
110
|
20.200.212.240/28
|
|
126
|
-
20.210.154.128/28
|
|
127
|
-
20.210.174.208/28
|
|
128
111
|
20.210.211.192/28
|
|
129
112
|
20.215.187.208/28
|
|
130
113
|
20.215.219.128/28
|
|
131
114
|
20.215.219.160/28
|
|
132
115
|
20.215.219.208/28
|
|
133
|
-
20.215.220.128/28
|
|
134
|
-
20.215.220.144/28
|
|
135
|
-
20.215.220.176/28
|
|
136
|
-
20.215.220.192/28
|
|
137
116
|
20.218.30.240/28
|
|
138
117
|
20.219.71.192/28
|
|
139
118
|
20.222.36.192/28
|
|
140
|
-
20.226.32.80/28
|
|
141
119
|
20.227.140.32/28
|
|
142
120
|
20.228.106.176/28
|
|
143
121
|
20.235.87.224/28
|
|
@@ -157,34 +135,24 @@ module Legitbot # :nodoc:
|
|
|
157
135
|
40.84.221.208/28
|
|
158
136
|
40.84.221.224/28
|
|
159
137
|
40.116.73.208/28
|
|
160
|
-
48.193.44.32/28
|
|
161
138
|
48.221.40.176/28
|
|
162
139
|
48.221.184.80/28
|
|
163
140
|
48.221.184.96/28
|
|
164
|
-
48.221.184.112/28
|
|
165
141
|
51.8.155.48/28
|
|
166
|
-
51.8.155.64/28
|
|
167
142
|
51.8.155.112/28
|
|
168
|
-
51.56.40.80/28
|
|
169
143
|
51.57.0.96/28
|
|
170
144
|
51.59.24.64/28
|
|
171
145
|
51.59.24.80/28
|
|
172
|
-
51.59.40.80/28
|
|
173
|
-
51.59.40.96/28
|
|
174
146
|
51.59.48.80/28
|
|
175
|
-
51.59.48.96/28
|
|
176
147
|
51.116.2.80/28
|
|
177
148
|
51.116.221.96/28
|
|
178
149
|
52.148.129.32/28
|
|
179
|
-
52.153.130.64/28
|
|
180
|
-
52.154.22.48/28
|
|
181
150
|
52.156.77.144/28
|
|
182
151
|
52.159.227.32/28
|
|
183
152
|
52.159.249.96/28
|
|
184
153
|
52.161.49.96/28
|
|
185
154
|
52.165.212.48/28
|
|
186
155
|
52.172.129.160/28
|
|
187
|
-
52.172.251.112/28
|
|
188
156
|
52.173.219.96/28
|
|
189
157
|
52.173.219.112/28
|
|
190
158
|
52.173.221.16/28
|
|
@@ -199,8 +167,6 @@ module Legitbot # :nodoc:
|
|
|
199
167
|
52.190.142.64/28
|
|
200
168
|
52.190.190.16/28
|
|
201
169
|
52.225.75.208/28
|
|
202
|
-
52.230.163.32/28
|
|
203
|
-
52.230.164.176/28
|
|
204
170
|
52.231.30.48/28
|
|
205
171
|
52.231.34.176/28
|
|
206
172
|
52.231.39.144/28
|
|
@@ -209,24 +175,19 @@ module Legitbot # :nodoc:
|
|
|
209
175
|
52.241.146.208/28
|
|
210
176
|
52.242.132.224/28
|
|
211
177
|
52.242.132.240/28
|
|
212
|
-
52.242.245.208/28
|
|
213
178
|
52.255.109.80/28
|
|
214
179
|
52.255.109.96/28
|
|
215
180
|
52.255.109.112/28
|
|
216
181
|
52.255.109.144/28
|
|
217
182
|
52.255.111.0/28
|
|
218
|
-
52.255.111.32/28
|
|
219
|
-
52.255.111.48/28
|
|
220
183
|
52.255.111.80/28
|
|
221
184
|
57.154.174.112/28
|
|
222
185
|
57.154.175.0/28
|
|
223
186
|
57.154.187.32/28
|
|
224
187
|
68.154.28.96/28
|
|
225
|
-
68.218.30.112/28
|
|
226
188
|
68.220.57.64/28
|
|
227
189
|
70.153.32.16/28
|
|
228
190
|
70.153.32.32/28
|
|
229
|
-
70.153.76.16/28
|
|
230
191
|
70.153.87.224/28
|
|
231
192
|
70.153.189.192/28
|
|
232
193
|
70.156.152.96/28
|
|
@@ -238,10 +199,8 @@ module Legitbot # :nodoc:
|
|
|
238
199
|
74.161.200.96/28
|
|
239
200
|
74.224.217.64/28
|
|
240
201
|
74.226.253.160/28
|
|
241
|
-
74.249.86.176/28
|
|
242
202
|
85.211.128.16/28
|
|
243
203
|
85.211.128.32/28
|
|
244
|
-
85.211.241.128/28
|
|
245
204
|
104.208.184.192/28
|
|
246
205
|
104.210.139.192/28
|
|
247
206
|
104.210.139.224/28
|
|
@@ -255,17 +214,14 @@ module Legitbot # :nodoc:
|
|
|
255
214
|
135.237.131.208/28
|
|
256
215
|
135.237.133.48/28
|
|
257
216
|
137.135.191.176/28
|
|
258
|
-
138.91.30.48/28
|
|
259
217
|
138.91.46.96/28
|
|
260
218
|
145.132.136.96/28
|
|
261
219
|
158.158.5.32/28
|
|
262
220
|
168.63.252.240/28
|
|
263
221
|
172.162.248.64/28
|
|
264
222
|
172.170.1.80/28
|
|
265
|
-
172.170.8.208/28
|
|
266
223
|
172.170.225.0/28
|
|
267
224
|
172.170.241.80/28
|
|
268
|
-
172.171.4.176/28
|
|
269
225
|
172.175.152.224/28
|
|
270
226
|
172.178.140.144/28
|
|
271
227
|
172.178.141.112/28
|
|
@@ -277,26 +233,15 @@ module Legitbot # :nodoc:
|
|
|
277
233
|
172.197.203.16/28
|
|
278
234
|
172.199.137.80/28
|
|
279
235
|
172.204.28.224/28
|
|
280
|
-
172.204.96.48/28
|
|
281
236
|
172.204.96.80/28
|
|
282
237
|
172.205.189.192/28
|
|
283
238
|
172.207.1.32/28
|
|
284
|
-
172.208.128.32/28
|
|
285
|
-
172.208.128.48/28
|
|
286
|
-
172.212.159.64/28
|
|
287
239
|
172.212.172.160/28
|
|
288
240
|
172.215.215.32/28
|
|
289
|
-
172.215.218.96/28
|
|
290
|
-
191.233.1.112/28
|
|
291
|
-
191.233.196.112/28
|
|
292
241
|
191.233.199.160/28
|
|
293
|
-
191.235.66.16/28
|
|
294
|
-
191.235.99.80/28
|
|
295
242
|
191.237.249.64/28
|
|
296
|
-
191.239.245.16/28
|
|
297
243
|
]
|
|
298
244
|
end
|
|
299
|
-
# rubocop:enable Metrics/ClassLength, Metrics/CollectionLiteralLength
|
|
300
245
|
|
|
301
246
|
# https://platform.openai.com/docs/bots
|
|
302
247
|
class OpenAISearch < BotMatch
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot.rb
CHANGED
data/test/amazon_bot_test.rb
CHANGED
|
@@ -30,7 +30,7 @@ class AmazonBotTest < Minitest::Test
|
|
|
30
30
|
refute_predicate bot, :valid?
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
# rubocop:disable Layout/LineLength
|
|
33
|
+
# rubocop:disable-next Layout/LineLength
|
|
34
34
|
def test_user_agent
|
|
35
35
|
bot = Legitbot.bot(
|
|
36
36
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -40,9 +40,8 @@ class AmazonBotTest < Minitest::Test
|
|
|
40
40
|
assert bot
|
|
41
41
|
assert_predicate bot, :valid?
|
|
42
42
|
end
|
|
43
|
-
# rubocop:enable Layout/LineLength
|
|
44
43
|
|
|
45
|
-
# rubocop:disable Layout/LineLength
|
|
44
|
+
# rubocop:disable-next Layout/LineLength
|
|
46
45
|
def test_valid_name
|
|
47
46
|
bot = Legitbot.bot(
|
|
48
47
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -51,9 +50,8 @@ class AmazonBotTest < Minitest::Test
|
|
|
51
50
|
|
|
52
51
|
assert_equal :amazonbot, bot.detected_as
|
|
53
52
|
end
|
|
54
|
-
# rubocop:enable Layout/LineLength
|
|
55
53
|
|
|
56
|
-
# rubocop:disable Layout/LineLength
|
|
54
|
+
# rubocop:disable-next Layout/LineLength
|
|
57
55
|
def test_fake_name
|
|
58
56
|
bot = Legitbot.bot(
|
|
59
57
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
|
|
@@ -62,5 +60,4 @@ class AmazonBotTest < Minitest::Test
|
|
|
62
60
|
|
|
63
61
|
assert_equal :amazonbot, bot.detected_as
|
|
64
62
|
end
|
|
65
|
-
# rubocop:enable Layout/LineLength
|
|
66
63
|
end
|
data/test/apple_test.rb
CHANGED
|
@@ -20,7 +20,7 @@ class AppleTest < Minitest::Test
|
|
|
20
20
|
assert_predicate match, :fake?
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
# rubocop:disable Layout/LineLength
|
|
23
|
+
# rubocop:disable-next Layout/LineLength
|
|
24
24
|
def test_user_agent
|
|
25
25
|
bot = Legitbot.bot(
|
|
26
26
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
|
|
@@ -30,5 +30,4 @@ class AppleTest < Minitest::Test
|
|
|
30
30
|
assert_equal :apple, bot.detected_as
|
|
31
31
|
assert_predicate bot, :valid?
|
|
32
32
|
end
|
|
33
|
-
# rubocop:enable Layout/LineLength
|
|
34
33
|
end
|
data/test/facebook_test.rb
CHANGED
|
@@ -42,7 +42,7 @@ class FacebookTest < Minitest::Test
|
|
|
42
42
|
end
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
# rubocop:disable Layout/LineLength
|
|
45
|
+
# rubocop:disable-next Layout/LineLength
|
|
46
46
|
def test_user_agent3
|
|
47
47
|
Legitbot.bot(
|
|
48
48
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
|
|
@@ -52,5 +52,4 @@ class FacebookTest < Minitest::Test
|
|
|
52
52
|
assert_predicate bot, :fake?
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
|
-
# rubocop:enable Layout/LineLength
|
|
56
55
|
end
|
|
@@ -4,7 +4,7 @@ require 'minitest/autorun'
|
|
|
4
4
|
require 'legitbot'
|
|
5
5
|
|
|
6
6
|
module Legitbot
|
|
7
|
-
# rubocop:disable Minitest/MultipleAssertions
|
|
7
|
+
# rubocop:disable-next Minitest/MultipleAssertions
|
|
8
8
|
module Validators
|
|
9
9
|
class NoRanges
|
|
10
10
|
include IpRanges
|
|
@@ -113,7 +113,7 @@ module Legitbot
|
|
|
113
113
|
refute FlattenRanges.valid_ip?('2a03:2880:f233:0:0:0:0:1')
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
-
# rubocop:disable Metrics/AbcSize
|
|
116
|
+
# rubocop:disable-next Metrics/AbcSize
|
|
117
117
|
def test_matcher_loader
|
|
118
118
|
assert_equal 0, LoadRanges.counter
|
|
119
119
|
assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
|
@@ -128,7 +128,6 @@ module Legitbot
|
|
|
128
128
|
assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
|
129
129
|
assert_equal 2, LoadRanges.counter
|
|
130
130
|
end
|
|
131
|
-
# rubocop:enable Metrics/AbcSize
|
|
132
131
|
|
|
133
132
|
def test_nil_ranges
|
|
134
133
|
assert NilRanges.valid_ip?('127.0.0.1')
|
|
@@ -139,5 +138,4 @@ module Legitbot
|
|
|
139
138
|
end
|
|
140
139
|
end
|
|
141
140
|
end
|
|
142
|
-
# rubocop:enable Minitest/MultipleAssertions
|
|
143
141
|
end
|
|
@@ -4,10 +4,9 @@ module Legitbot
|
|
|
4
4
|
module MetaIpRanges
|
|
5
5
|
alias fetch_ip_ranges_orig fetch_ip_ranges
|
|
6
6
|
|
|
7
|
-
# rubocop:disable Layout/LineLength
|
|
7
|
+
# rubocop:disable-next Layout/LineLength
|
|
8
8
|
def fetch_ip_ranges
|
|
9
9
|
['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24']
|
|
10
10
|
end
|
|
11
|
-
# rubocop:enable Layout/LineLength
|
|
12
11
|
end
|
|
13
12
|
end
|
data/test/meta_test.rb
CHANGED
|
@@ -65,7 +65,7 @@ class MetaTest < Minitest::Test
|
|
|
65
65
|
end
|
|
66
66
|
end
|
|
67
67
|
|
|
68
|
-
# rubocop:disable Layout/LineLength
|
|
68
|
+
# rubocop:disable-next Layout/LineLength
|
|
69
69
|
def test_user_agent4
|
|
70
70
|
Legitbot.bot(
|
|
71
71
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 meta-externalagent/1.1 Twitterbot/1.0',
|
|
@@ -75,5 +75,4 @@ class MetaTest < Minitest::Test
|
|
|
75
75
|
assert_predicate bot, :fake?
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
|
-
# rubocop:enable Layout/LineLength
|
|
79
78
|
end
|
data/test/pinterest_test.rb
CHANGED
|
@@ -40,7 +40,7 @@ class PinterestTest < Minitest::Test
|
|
|
40
40
|
assert_predicate bot, :valid?
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
# rubocop:disable Layout/LineLength
|
|
43
|
+
# rubocop:disable-next Layout/LineLength
|
|
44
44
|
def test_android_not_bot
|
|
45
45
|
bot = Legitbot.bot(
|
|
46
46
|
'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]',
|
|
@@ -49,7 +49,6 @@ class PinterestTest < Minitest::Test
|
|
|
49
49
|
|
|
50
50
|
assert_nil bot
|
|
51
51
|
end
|
|
52
|
-
# rubocop:enable Layout/LineLength
|
|
53
52
|
|
|
54
53
|
def test_engine_name
|
|
55
54
|
bot = Legitbot.bot(
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legitbot
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.14.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexander Azarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-08-
|
|
11
|
+
date: 2026-08-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fast_interval_tree
|
|
@@ -93,6 +93,7 @@ files:
|
|
|
93
93
|
- lib/legitbot.rb
|
|
94
94
|
- lib/legitbot/ahrefs.rb
|
|
95
95
|
- lib/legitbot/amazon.rb
|
|
96
|
+
- lib/legitbot/anthropic.rb
|
|
96
97
|
- lib/legitbot/apple.rb
|
|
97
98
|
- lib/legitbot/baidu.rb
|
|
98
99
|
- lib/legitbot/bing.rb
|