legitbot 1.7.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/build.yml +1 -1
- data/Gemfile +13 -0
- data/README.md +1 -0
- data/legitbot.gemspec +0 -9
- data/lib/legitbot/ias.rb +16 -0
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot.rb +1 -0
- data/lib/rubocop/cop/custom/ip_ranges.rb +10 -4
- data/test/ahrefs_test.rb +4 -0
- data/test/alexa_test.rb +4 -0
- data/test/amazon_test.rb +6 -0
- data/test/apple_test.rb +3 -0
- data/test/facebook_test.rb +3 -0
- data/test/google_test.rb +6 -0
- data/test/ias_test.rb +39 -0
- data/test/legitbot/validators/ip_ranges_test.rb +5 -0
- data/test/oracle_test.rb +4 -0
- data/test/petalbot_test.rb +6 -0
- data/test/pinterest_test.rb +6 -0
- data/test/twitter_test.rb +4 -0
- data/test/you_test.rb +4 -0
- metadata +4 -182
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a7cde94cf9e8a396867e4e97a490c1b4da0b300652e619ac200477a5e3aed1d5
|
4
|
+
data.tar.gz: 1d505df51aa086231f85080fcf3a60291a660cfce3ce79bdeab0591ba9d27578
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41f811fd8c20c9a442218e36a8e54e1e72731b8443c6141205b5f3b7accbdbaeb85491134d2ca3f4cb817d01dc774348c52dae705a7655d98345f975782b5c4a
|
7
|
+
data.tar.gz: 77912f09be50c5d868099a6ecc2402b582d17039a82b9a57ee790724bf3b99123d7b5db071928f944c814afcc226818530b953c62f31d6f4a5d1df6bfdcefc54
|
data/.github/workflows/build.yml
CHANGED
data/Gemfile
CHANGED
@@ -2,3 +2,16 @@
|
|
2
2
|
|
3
3
|
source 'https://rubygems.org'
|
4
4
|
gemspec
|
5
|
+
|
6
|
+
group :development do
|
7
|
+
gem 'bump'
|
8
|
+
gem 'dns_mock'
|
9
|
+
gem 'jsonpath'
|
10
|
+
gem 'minitest'
|
11
|
+
gem 'minitest-hooks'
|
12
|
+
gem 'nokogiri'
|
13
|
+
gem 'rake'
|
14
|
+
gem 'rubocop'
|
15
|
+
gem 'rubocop-minitest'
|
16
|
+
gem 'simplecov-cobertura'
|
17
|
+
end
|
data/README.md
CHANGED
@@ -57,6 +57,7 @@ end
|
|
57
57
|
* [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
|
58
58
|
* [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
|
59
59
|
* [Google crawlers](https://support.google.com/webmasters/answer/1061943)
|
60
|
+
* [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
|
60
61
|
* [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
|
61
62
|
* [Petal search engine](http://aspiegel.com/petalbot)
|
62
63
|
* [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
|
data/legitbot.gemspec
CHANGED
@@ -22,15 +22,6 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.required_ruby_version = '>= 2.6.0'
|
23
23
|
spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
|
24
24
|
spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
|
25
|
-
spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
|
26
|
-
spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
|
27
|
-
spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
|
28
|
-
spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
|
29
|
-
spec.add_development_dependency 'nokogiri', '~> 1.13.0', '>= 1.13.0'
|
30
|
-
spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
|
31
|
-
spec.add_development_dependency 'rubocop', '~> 1.36.0', '>= 1.36.0'
|
32
|
-
spec.add_development_dependency 'rubocop-minitest', '~> 0.22.0', '>= 0.22.0'
|
33
|
-
spec.add_development_dependency 'simplecov-cobertura', '~> 2.0', '>= 2.0'
|
34
25
|
|
35
26
|
spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
36
27
|
spec.rdoc_options = ['--charset=UTF-8']
|
data/lib/legitbot/ias.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Legitbot # :nodoc:
|
4
|
+
# https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/
|
5
|
+
class Ias < BotMatch
|
6
|
+
# @fetch:url https://integralads.com/policy-docs/iasbot.json
|
7
|
+
# @fetch:jsonpath $.publicIPs[*].ipv4
|
8
|
+
ip_ranges %w[
|
9
|
+
3.217.168.199
|
10
|
+
3.226.51.67
|
11
|
+
18.214.43.70
|
12
|
+
]
|
13
|
+
end
|
14
|
+
|
15
|
+
rule Legitbot::Ias, %w[ias_crawler ias_wombles]
|
16
|
+
end
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot.rb
CHANGED
@@ -12,6 +12,7 @@ require_relative 'legitbot/bing'
|
|
12
12
|
require_relative 'legitbot/duckduckgo'
|
13
13
|
require_relative 'legitbot/facebook'
|
14
14
|
require_relative 'legitbot/google'
|
15
|
+
require_relative 'legitbot/ias'
|
15
16
|
require_relative 'legitbot/oracle'
|
16
17
|
require_relative 'legitbot/petalbot'
|
17
18
|
require_relative 'legitbot/pinterest'
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'ipaddr'
|
4
4
|
require 'net/http'
|
5
5
|
require 'nokogiri'
|
6
|
+
require 'jsonpath'
|
6
7
|
require 'rubocop'
|
7
8
|
require 'uri'
|
8
9
|
|
@@ -34,12 +35,17 @@ module RuboCop
|
|
34
35
|
|
35
36
|
private
|
36
37
|
|
37
|
-
def fetch_ips(url:, selector:)
|
38
|
+
def fetch_ips(url:, selector: nil, jsonpath: nil)
|
38
39
|
response = Net::HTTP.get_response URI(url)
|
39
40
|
response.value
|
40
41
|
|
41
|
-
|
42
|
-
|
42
|
+
if selector
|
43
|
+
document = Nokogiri::HTML response.body
|
44
|
+
document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
|
45
|
+
else
|
46
|
+
document = JSON.parse response.body
|
47
|
+
JsonPath.new(jsonpath).on(document).sort_by(&IPAddr.method(:new))
|
48
|
+
end
|
43
49
|
end
|
44
50
|
|
45
51
|
def read_node_ips(value)
|
@@ -54,7 +60,7 @@ module RuboCop
|
|
54
60
|
end
|
55
61
|
|
56
62
|
def mandatory_params?(params)
|
57
|
-
params.include?(:url) && params.include?(:selector)
|
63
|
+
params.include?(:url) && (params.include?(:selector) || params.include?(:jsonpath))
|
58
64
|
end
|
59
65
|
|
60
66
|
def fetch_params(node)
|
data/test/ahrefs_test.rb
CHANGED
@@ -9,12 +9,14 @@ class AhrefsTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Ahrefs.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '54.36.148.0'
|
17
18
|
match = Legitbot::Ahrefs.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class AhrefsTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class AhrefsTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
|
33
36
|
'54.36.148.0'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
data/test/alexa_test.rb
CHANGED
@@ -9,12 +9,14 @@ class AlexaTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Alexa.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '52.86.176.3'
|
17
18
|
match = Legitbot::Alexa.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class AlexaTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class AlexaTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
|
33
36
|
'52.86.176.3'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
data/test/amazon_test.rb
CHANGED
@@ -9,12 +9,14 @@ class AmazonTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Amazon.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '54.166.7.90'
|
17
18
|
match = Legitbot::Amazon.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class AmazonTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class AmazonTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
33
36
|
'54.166.7.90'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
@@ -41,6 +45,7 @@ class AmazonTest < Minitest::Test
|
|
41
45
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
42
46
|
'54.166.7.90'
|
43
47
|
)
|
48
|
+
|
44
49
|
assert_equal :amazon, bot.detected_as
|
45
50
|
end
|
46
51
|
|
@@ -49,6 +54,7 @@ class AmazonTest < Minitest::Test
|
|
49
54
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
50
55
|
'81.1.172.108'
|
51
56
|
)
|
57
|
+
|
52
58
|
assert_equal :amazon, bot.detected_as
|
53
59
|
end
|
54
60
|
end
|
data/test/apple_test.rb
CHANGED
@@ -9,12 +9,14 @@ class AppleTest < Minitest::Test
|
|
9
9
|
def test_valid_ip
|
10
10
|
ip = '17.58.98.60'
|
11
11
|
match = Legitbot::Apple.new(ip)
|
12
|
+
|
12
13
|
assert_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_invalid_ip
|
16
17
|
ip = '127.0.0.1'
|
17
18
|
match = Legitbot::Apple.new(ip)
|
19
|
+
|
18
20
|
assert_predicate match, :fake?
|
19
21
|
end
|
20
22
|
|
@@ -24,6 +26,7 @@ class AppleTest < Minitest::Test
|
|
24
26
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
|
25
27
|
'17.58.98.60'
|
26
28
|
)
|
29
|
+
|
27
30
|
assert_equal :apple, bot.detected_as
|
28
31
|
assert_predicate bot, :valid?
|
29
32
|
end
|
data/test/facebook_test.rb
CHANGED
@@ -19,16 +19,19 @@ class FacebookTest < Minitest::Test
|
|
19
19
|
def test_valid_ip
|
20
20
|
ip = '69.63.186.89'
|
21
21
|
match = Legitbot::Facebook.new(ip)
|
22
|
+
|
22
23
|
assert_predicate match, :valid?
|
23
24
|
|
24
25
|
ip = '69.171.251.1'
|
25
26
|
match = Legitbot::Facebook.new(ip)
|
27
|
+
|
26
28
|
assert_predicate match, :valid?
|
27
29
|
end
|
28
30
|
|
29
31
|
def test_invalid_ip
|
30
32
|
ip = '127.0.0.1'
|
31
33
|
match = Legitbot::Facebook.new(ip)
|
34
|
+
|
32
35
|
assert_predicate match, :fake?
|
33
36
|
end
|
34
37
|
|
data/test/google_test.rb
CHANGED
@@ -9,12 +9,14 @@ class GoogleTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Google.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '66.249.64.141'
|
17
18
|
match = Legitbot::Google.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class GoogleTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class GoogleTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
33
36
|
'66.249.64.141'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
@@ -41,6 +45,7 @@ class GoogleTest < Minitest::Test
|
|
41
45
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
42
46
|
'66.249.64.141'
|
43
47
|
)
|
48
|
+
|
44
49
|
assert_equal :google, bot.detected_as
|
45
50
|
end
|
46
51
|
|
@@ -49,6 +54,7 @@ class GoogleTest < Minitest::Test
|
|
49
54
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
50
55
|
'81.1.172.108'
|
51
56
|
)
|
57
|
+
|
52
58
|
assert_equal :google, bot.detected_as
|
53
59
|
end
|
54
60
|
end
|
data/test/ias_test.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'test_helper'
|
4
|
+
|
5
|
+
class IasTest < Minitest::Test
|
6
|
+
def test_malicious_ip
|
7
|
+
ip = '149.210.164.47'
|
8
|
+
match = Legitbot::Ias.new ip
|
9
|
+
|
10
|
+
refute_predicate match, :valid?
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_valid_ip
|
14
|
+
ip = '18.214.43.70'
|
15
|
+
match = Legitbot::Ias.new ip
|
16
|
+
|
17
|
+
assert_predicate match, :valid?
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_malicious_ua
|
21
|
+
bot = Legitbot.bot(
|
22
|
+
'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
|
23
|
+
'18.214.43.72'
|
24
|
+
)
|
25
|
+
|
26
|
+
assert bot
|
27
|
+
refute_predicate bot, :valid?
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_valid_ua
|
31
|
+
bot = Legitbot.bot(
|
32
|
+
'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
|
33
|
+
'18.214.43.70'
|
34
|
+
)
|
35
|
+
|
36
|
+
assert bot
|
37
|
+
assert_predicate bot, :valid?
|
38
|
+
end
|
39
|
+
end
|
@@ -60,17 +60,20 @@ module Legitbot
|
|
60
60
|
class IpRangesTest < Minitest::Test
|
61
61
|
def test_partition_method
|
62
62
|
empty = NoRanges.partition_ips([])
|
63
|
+
|
63
64
|
assert_empty empty
|
64
65
|
end
|
65
66
|
|
66
67
|
def test_ipv6_partition
|
67
68
|
ipv6 = NoRanges.partition_ips(['2a03:2880:f234::/48'])
|
69
|
+
|
68
70
|
assert_nil ipv6[:ipv4].top_node
|
69
71
|
refute_nil ipv6[:ipv6].top_node
|
70
72
|
end
|
71
73
|
|
72
74
|
def test_ipv4_partition
|
73
75
|
ipv4 = NoRanges.partition_ips(['66.220.144.0/21'])
|
76
|
+
|
74
77
|
refute_nil ipv4[:ipv4].top_node
|
75
78
|
assert_nil ipv4[:ipv6].top_node
|
76
79
|
end
|
@@ -87,6 +90,7 @@ module Legitbot
|
|
87
90
|
assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
|
88
91
|
|
89
92
|
matcher = EmptyRanges.new '127.0.0.0'
|
93
|
+
|
90
94
|
assert_predicate matcher, :valid_ip?
|
91
95
|
end
|
92
96
|
|
@@ -113,6 +117,7 @@ module Legitbot
|
|
113
117
|
assert_equal 1, LoadRanges.counter
|
114
118
|
|
115
119
|
LoadRanges.reload_ips
|
120
|
+
|
116
121
|
refute_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
117
122
|
refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
|
118
123
|
assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
data/test/oracle_test.rb
CHANGED
@@ -6,12 +6,14 @@ class OracleTest < Minitest::Test
|
|
6
6
|
def test_malicious_ip
|
7
7
|
ip = '149.210.164.47'
|
8
8
|
match = Legitbot::Oracle.new ip
|
9
|
+
|
9
10
|
refute_predicate match, :valid?
|
10
11
|
end
|
11
12
|
|
12
13
|
def test_valid_ip
|
13
14
|
ip = '148.64.56.64'
|
14
15
|
match = Legitbot::Oracle.new ip
|
16
|
+
|
15
17
|
assert_predicate match, :valid?
|
16
18
|
end
|
17
19
|
|
@@ -20,6 +22,7 @@ class OracleTest < Minitest::Test
|
|
20
22
|
'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
|
21
23
|
'149.210.164.47'
|
22
24
|
)
|
25
|
+
|
23
26
|
assert bot
|
24
27
|
refute_predicate bot, :valid?
|
25
28
|
end
|
@@ -29,6 +32,7 @@ class OracleTest < Minitest::Test
|
|
29
32
|
'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
|
30
33
|
'148.64.56.64'
|
31
34
|
)
|
35
|
+
|
32
36
|
assert bot
|
33
37
|
assert_predicate bot, :valid?
|
34
38
|
end
|
data/test/petalbot_test.rb
CHANGED
@@ -9,12 +9,14 @@ class PetalbotTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Petalbot.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '114.119.134.10'
|
17
18
|
match = Legitbot::Petalbot.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class PetalbotTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class PetalbotTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
|
33
36
|
'114.119.134.10'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
@@ -41,6 +45,7 @@ class PetalbotTest < Minitest::Test
|
|
41
45
|
'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
|
42
46
|
'66.249.64.141'
|
43
47
|
)
|
48
|
+
|
44
49
|
assert_equal :petalbot, bot.detected_as
|
45
50
|
end
|
46
51
|
|
@@ -49,6 +54,7 @@ class PetalbotTest < Minitest::Test
|
|
49
54
|
'Mozilla/5.0 (compatible; PetalBot/2.1; +http://www.google.com/bot.html)',
|
50
55
|
'81.1.172.108'
|
51
56
|
)
|
57
|
+
|
52
58
|
assert_equal :petalbot, bot.detected_as
|
53
59
|
end
|
54
60
|
end
|
data/test/pinterest_test.rb
CHANGED
@@ -9,12 +9,14 @@ class PinterestTest < Minitest::Test
|
|
9
9
|
def test_malicious_ip
|
10
10
|
ip = '149.210.164.47'
|
11
11
|
match = Legitbot::Pinterest.new ip
|
12
|
+
|
12
13
|
refute_predicate match, :valid?
|
13
14
|
end
|
14
15
|
|
15
16
|
def test_valid_ip
|
16
17
|
ip = '54.236.1.11'
|
17
18
|
match = Legitbot::Pinterest.new ip
|
19
|
+
|
18
20
|
assert_predicate match, :valid?
|
19
21
|
end
|
20
22
|
|
@@ -23,6 +25,7 @@ class PinterestTest < Minitest::Test
|
|
23
25
|
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
|
24
26
|
'149.210.164.47'
|
25
27
|
)
|
28
|
+
|
26
29
|
assert bot
|
27
30
|
refute_predicate bot, :valid?
|
28
31
|
end
|
@@ -32,6 +35,7 @@ class PinterestTest < Minitest::Test
|
|
32
35
|
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
|
33
36
|
'54.236.1.11'
|
34
37
|
)
|
38
|
+
|
35
39
|
assert bot
|
36
40
|
assert_predicate bot, :valid?
|
37
41
|
end
|
@@ -42,6 +46,7 @@ class PinterestTest < Minitest::Test
|
|
42
46
|
'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]',
|
43
47
|
'85.117.106.133'
|
44
48
|
)
|
49
|
+
|
45
50
|
assert_nil bot
|
46
51
|
end
|
47
52
|
# rubocop:enable Layout/LineLength
|
@@ -51,6 +56,7 @@ class PinterestTest < Minitest::Test
|
|
51
56
|
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
|
52
57
|
'54.236.1.11'
|
53
58
|
)
|
59
|
+
|
54
60
|
assert_equal :pinterest, bot.detected_as
|
55
61
|
end
|
56
62
|
end
|
data/test/twitter_test.rb
CHANGED
@@ -6,12 +6,14 @@ class TwitterTest < Minitest::Test
|
|
6
6
|
def test_malicious_ip
|
7
7
|
ip = '149.210.164.47'
|
8
8
|
match = Legitbot::Twitter.new ip
|
9
|
+
|
9
10
|
refute_predicate match, :valid?
|
10
11
|
end
|
11
12
|
|
12
13
|
def test_valid_ip
|
13
14
|
ip = '199.16.156.125'
|
14
15
|
match = Legitbot::Twitter.new ip
|
16
|
+
|
15
17
|
assert_predicate match, :valid?
|
16
18
|
end
|
17
19
|
|
@@ -20,6 +22,7 @@ class TwitterTest < Minitest::Test
|
|
20
22
|
'Twitterbot/1.0',
|
21
23
|
'149.210.164.47'
|
22
24
|
)
|
25
|
+
|
23
26
|
assert bot
|
24
27
|
refute_predicate bot, :valid?
|
25
28
|
end
|
@@ -29,6 +32,7 @@ class TwitterTest < Minitest::Test
|
|
29
32
|
'Twitterbot/1.0',
|
30
33
|
'199.16.156.125'
|
31
34
|
)
|
35
|
+
|
32
36
|
assert bot
|
33
37
|
assert_predicate bot, :valid?
|
34
38
|
end
|
data/test/you_test.rb
CHANGED
@@ -6,12 +6,14 @@ class YouTest < Minitest::Test
|
|
6
6
|
def test_malicious_ip
|
7
7
|
ip = '20.59.41.22'
|
8
8
|
match = Legitbot::You.new ip
|
9
|
+
|
9
10
|
refute_predicate match, :valid?
|
10
11
|
end
|
11
12
|
|
12
13
|
def test_valid_ip
|
13
14
|
ip = '20.59.40.22'
|
14
15
|
match = Legitbot::You.new ip
|
16
|
+
|
15
17
|
assert_predicate match, :valid?
|
16
18
|
end
|
17
19
|
|
@@ -20,6 +22,7 @@ class YouTest < Minitest::Test
|
|
20
22
|
'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
|
21
23
|
'20.59.41.22'
|
22
24
|
)
|
25
|
+
|
23
26
|
assert bot
|
24
27
|
refute_predicate bot, :valid?
|
25
28
|
end
|
@@ -29,6 +32,7 @@ class YouTest < Minitest::Test
|
|
29
32
|
'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
|
30
33
|
'20.59.40.22'
|
31
34
|
)
|
35
|
+
|
32
36
|
assert bot
|
33
37
|
assert_predicate bot, :valid?
|
34
38
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: legitbot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Azarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-07-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fast_interval_tree
|
@@ -50,186 +50,6 @@ dependencies:
|
|
50
50
|
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 0.2.1
|
53
|
-
- !ruby/object:Gem::Dependency
|
54
|
-
name: bump
|
55
|
-
requirement: !ruby/object:Gem::Requirement
|
56
|
-
requirements:
|
57
|
-
- - "~>"
|
58
|
-
- !ruby/object:Gem::Version
|
59
|
-
version: '0.8'
|
60
|
-
- - ">="
|
61
|
-
- !ruby/object:Gem::Version
|
62
|
-
version: 0.8.0
|
63
|
-
type: :development
|
64
|
-
prerelease: false
|
65
|
-
version_requirements: !ruby/object:Gem::Requirement
|
66
|
-
requirements:
|
67
|
-
- - "~>"
|
68
|
-
- !ruby/object:Gem::Version
|
69
|
-
version: '0.8'
|
70
|
-
- - ">="
|
71
|
-
- !ruby/object:Gem::Version
|
72
|
-
version: 0.8.0
|
73
|
-
- !ruby/object:Gem::Dependency
|
74
|
-
name: dns_mock
|
75
|
-
requirement: !ruby/object:Gem::Requirement
|
76
|
-
requirements:
|
77
|
-
- - "~>"
|
78
|
-
- !ruby/object:Gem::Version
|
79
|
-
version: 1.5.0
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: 1.5.0
|
83
|
-
type: :development
|
84
|
-
prerelease: false
|
85
|
-
version_requirements: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: 1.5.0
|
90
|
-
- - ">="
|
91
|
-
- !ruby/object:Gem::Version
|
92
|
-
version: 1.5.0
|
93
|
-
- !ruby/object:Gem::Dependency
|
94
|
-
name: minitest
|
95
|
-
requirement: !ruby/object:Gem::Requirement
|
96
|
-
requirements:
|
97
|
-
- - "~>"
|
98
|
-
- !ruby/object:Gem::Version
|
99
|
-
version: '5.1'
|
100
|
-
- - ">="
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: 5.1.0
|
103
|
-
type: :development
|
104
|
-
prerelease: false
|
105
|
-
version_requirements: !ruby/object:Gem::Requirement
|
106
|
-
requirements:
|
107
|
-
- - "~>"
|
108
|
-
- !ruby/object:Gem::Version
|
109
|
-
version: '5.1'
|
110
|
-
- - ">="
|
111
|
-
- !ruby/object:Gem::Version
|
112
|
-
version: 5.1.0
|
113
|
-
- !ruby/object:Gem::Dependency
|
114
|
-
name: minitest-hooks
|
115
|
-
requirement: !ruby/object:Gem::Requirement
|
116
|
-
requirements:
|
117
|
-
- - "~>"
|
118
|
-
- !ruby/object:Gem::Version
|
119
|
-
version: '1.5'
|
120
|
-
- - ">="
|
121
|
-
- !ruby/object:Gem::Version
|
122
|
-
version: 1.5.0
|
123
|
-
type: :development
|
124
|
-
prerelease: false
|
125
|
-
version_requirements: !ruby/object:Gem::Requirement
|
126
|
-
requirements:
|
127
|
-
- - "~>"
|
128
|
-
- !ruby/object:Gem::Version
|
129
|
-
version: '1.5'
|
130
|
-
- - ">="
|
131
|
-
- !ruby/object:Gem::Version
|
132
|
-
version: 1.5.0
|
133
|
-
- !ruby/object:Gem::Dependency
|
134
|
-
name: nokogiri
|
135
|
-
requirement: !ruby/object:Gem::Requirement
|
136
|
-
requirements:
|
137
|
-
- - "~>"
|
138
|
-
- !ruby/object:Gem::Version
|
139
|
-
version: 1.13.0
|
140
|
-
- - ">="
|
141
|
-
- !ruby/object:Gem::Version
|
142
|
-
version: 1.13.0
|
143
|
-
type: :development
|
144
|
-
prerelease: false
|
145
|
-
version_requirements: !ruby/object:Gem::Requirement
|
146
|
-
requirements:
|
147
|
-
- - "~>"
|
148
|
-
- !ruby/object:Gem::Version
|
149
|
-
version: 1.13.0
|
150
|
-
- - ">="
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: 1.13.0
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: rake
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - "~>"
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: '13.0'
|
160
|
-
- - ">="
|
161
|
-
- !ruby/object:Gem::Version
|
162
|
-
version: 13.0.0
|
163
|
-
type: :development
|
164
|
-
prerelease: false
|
165
|
-
version_requirements: !ruby/object:Gem::Requirement
|
166
|
-
requirements:
|
167
|
-
- - "~>"
|
168
|
-
- !ruby/object:Gem::Version
|
169
|
-
version: '13.0'
|
170
|
-
- - ">="
|
171
|
-
- !ruby/object:Gem::Version
|
172
|
-
version: 13.0.0
|
173
|
-
- !ruby/object:Gem::Dependency
|
174
|
-
name: rubocop
|
175
|
-
requirement: !ruby/object:Gem::Requirement
|
176
|
-
requirements:
|
177
|
-
- - "~>"
|
178
|
-
- !ruby/object:Gem::Version
|
179
|
-
version: 1.36.0
|
180
|
-
- - ">="
|
181
|
-
- !ruby/object:Gem::Version
|
182
|
-
version: 1.36.0
|
183
|
-
type: :development
|
184
|
-
prerelease: false
|
185
|
-
version_requirements: !ruby/object:Gem::Requirement
|
186
|
-
requirements:
|
187
|
-
- - "~>"
|
188
|
-
- !ruby/object:Gem::Version
|
189
|
-
version: 1.36.0
|
190
|
-
- - ">="
|
191
|
-
- !ruby/object:Gem::Version
|
192
|
-
version: 1.36.0
|
193
|
-
- !ruby/object:Gem::Dependency
|
194
|
-
name: rubocop-minitest
|
195
|
-
requirement: !ruby/object:Gem::Requirement
|
196
|
-
requirements:
|
197
|
-
- - "~>"
|
198
|
-
- !ruby/object:Gem::Version
|
199
|
-
version: 0.22.0
|
200
|
-
- - ">="
|
201
|
-
- !ruby/object:Gem::Version
|
202
|
-
version: 0.22.0
|
203
|
-
type: :development
|
204
|
-
prerelease: false
|
205
|
-
version_requirements: !ruby/object:Gem::Requirement
|
206
|
-
requirements:
|
207
|
-
- - "~>"
|
208
|
-
- !ruby/object:Gem::Version
|
209
|
-
version: 0.22.0
|
210
|
-
- - ">="
|
211
|
-
- !ruby/object:Gem::Version
|
212
|
-
version: 0.22.0
|
213
|
-
- !ruby/object:Gem::Dependency
|
214
|
-
name: simplecov-cobertura
|
215
|
-
requirement: !ruby/object:Gem::Requirement
|
216
|
-
requirements:
|
217
|
-
- - "~>"
|
218
|
-
- !ruby/object:Gem::Version
|
219
|
-
version: '2.0'
|
220
|
-
- - ">="
|
221
|
-
- !ruby/object:Gem::Version
|
222
|
-
version: '2.0'
|
223
|
-
type: :development
|
224
|
-
prerelease: false
|
225
|
-
version_requirements: !ruby/object:Gem::Requirement
|
226
|
-
requirements:
|
227
|
-
- - "~>"
|
228
|
-
- !ruby/object:Gem::Version
|
229
|
-
version: '2.0'
|
230
|
-
- - ">="
|
231
|
-
- !ruby/object:Gem::Version
|
232
|
-
version: '2.0'
|
233
53
|
description: Does Web request come from a real search engine or from an impersonating
|
234
54
|
agent?
|
235
55
|
email: self@alaz.me
|
@@ -261,6 +81,7 @@ files:
|
|
261
81
|
- lib/legitbot/duckduckgo.rb
|
262
82
|
- lib/legitbot/facebook.rb
|
263
83
|
- lib/legitbot/google.rb
|
84
|
+
- lib/legitbot/ias.rb
|
264
85
|
- lib/legitbot/legitbot.rb
|
265
86
|
- lib/legitbot/oracle.rb
|
266
87
|
- lib/legitbot/petalbot.rb
|
@@ -284,6 +105,7 @@ files:
|
|
284
105
|
- test/botmatch_test.rb
|
285
106
|
- test/facebook_test.rb
|
286
107
|
- test/google_test.rb
|
108
|
+
- test/ias_test.rb
|
287
109
|
- test/legitbot/validators/domains_test.rb
|
288
110
|
- test/legitbot/validators/ip_ranges_test.rb
|
289
111
|
- test/legitbot_test.rb
|