legitbot 1.6.0 → 1.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/autocorrect.yml +44 -0
- data/.github/workflows/build.yml +4 -6
- data/.rubocop.yml +5 -1
- data/README.md +1 -0
- data/legitbot.gemspec +3 -3
- data/lib/legitbot/duckduckgo.rb +11 -11
- data/lib/legitbot/pinterest.rb +1 -1
- data/lib/legitbot/version.rb +1 -1
- data/lib/legitbot/you.rb +12 -0
- data/lib/legitbot.rb +2 -1
- data/lib/rubocop/cop/custom/ip_ranges.rb +77 -0
- data/lib/rubocop/cop/custom.rb +3 -0
- data/test/ahrefs_test.rb +4 -4
- data/test/alexa_test.rb +4 -4
- data/test/amazon_test.rb +4 -4
- data/test/apple_test.rb +3 -3
- data/test/facebook_test.rb +6 -6
- data/test/google_test.rb +4 -4
- data/test/legitbot/validators/ip_ranges_test.rb +7 -7
- data/test/oracle_test.rb +4 -4
- data/test/petalbot_test.rb +4 -4
- data/test/pinterest_test.rb +4 -4
- data/test/twitter_test.rb +4 -4
- data/test/you_test.rb +36 -0
- metadata +36 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 929689d8f5882d4c13a3d03175a2a6197f11ce04c6512caa4312dc67a927e8a1
|
4
|
+
data.tar.gz: 809f94a123ca7f43b8ad0fda1421ee01e442b42a1c8e28e2e12c0e23ad4d0ca5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e40267f56b07a22f719f91a8b2c1b43b4e8b81e599b8aef854a72c2eba804d7c5ffecf1916fdf812fc6c772cc7aa854878f4939ba46550001b7f6a4b351f855e
|
7
|
+
data.tar.gz: edf9fc3ba2b87c20cec3b87d08d20c3c304702fcd81098a41d43f4b40d747eac79197251069db0453213adfd10d3d714d42df99bedeb2d5072d5fe66520531d3
|
@@ -0,0 +1,44 @@
|
|
1
|
+
name: autocorrect
|
2
|
+
|
3
|
+
on:
|
4
|
+
schedule:
|
5
|
+
- cron: '29 6 * * 6'
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
lint:
|
9
|
+
runs-on: ubuntu-latest
|
10
|
+
|
11
|
+
strategy:
|
12
|
+
matrix:
|
13
|
+
ruby: [ 3.0 ]
|
14
|
+
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v3
|
17
|
+
- name: Set up Ruby
|
18
|
+
uses: ruby/setup-ruby@v1
|
19
|
+
with:
|
20
|
+
ruby-version: ${{ matrix.ruby }}
|
21
|
+
- name: Cache dependencies
|
22
|
+
uses: actions/cache@v3
|
23
|
+
with:
|
24
|
+
path: vendor/bundle
|
25
|
+
key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
|
26
|
+
restore-keys: |
|
27
|
+
${{ runner.os }}-${{ matrix.ruby }}-gems-
|
28
|
+
- name: Install dependencies
|
29
|
+
run: |
|
30
|
+
bundle config path vendor/bundle
|
31
|
+
bundle install --jobs 4 --retry 3
|
32
|
+
- name: Run tests
|
33
|
+
run: bundle exec rake test
|
34
|
+
- name: Run linter
|
35
|
+
run: bundle exec rubocop --auto-correct
|
36
|
+
- name: Create Pull Request
|
37
|
+
uses: peter-evans/create-pull-request@v3
|
38
|
+
with:
|
39
|
+
branch: update/lint-autocorrect
|
40
|
+
delete-branch: true
|
41
|
+
commit-message: '🔧 autocorrect by Rubocop'
|
42
|
+
title: '[rubocop] Automatic corrections'
|
43
|
+
body: |
|
44
|
+
Please review automatic changes proposed by `rubocop -a`.
|
data/.github/workflows/build.yml
CHANGED
@@ -3,8 +3,6 @@ name: build
|
|
3
3
|
on:
|
4
4
|
pull_request:
|
5
5
|
push:
|
6
|
-
schedule:
|
7
|
-
- cron: '29 6 * * 6'
|
8
6
|
|
9
7
|
jobs:
|
10
8
|
test:
|
@@ -16,13 +14,13 @@ jobs:
|
|
16
14
|
ruby: [ jruby, 2.6, 2.7, 3.0 ]
|
17
15
|
|
18
16
|
steps:
|
19
|
-
- uses: actions/checkout@
|
17
|
+
- uses: actions/checkout@v3
|
20
18
|
- name: Set up Ruby
|
21
19
|
uses: ruby/setup-ruby@v1
|
22
20
|
with:
|
23
21
|
ruby-version: ${{ matrix.ruby }}
|
24
22
|
- name: Cache dependencies
|
25
|
-
uses: actions/cache@
|
23
|
+
uses: actions/cache@v3
|
26
24
|
with:
|
27
25
|
path: vendor/bundle
|
28
26
|
key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
|
@@ -44,13 +42,13 @@ jobs:
|
|
44
42
|
ruby: [ 3.0 ]
|
45
43
|
|
46
44
|
steps:
|
47
|
-
- uses: actions/checkout@
|
45
|
+
- uses: actions/checkout@v3
|
48
46
|
- name: Set up Ruby
|
49
47
|
uses: ruby/setup-ruby@v1
|
50
48
|
with:
|
51
49
|
ruby-version: ${{ matrix.ruby }}
|
52
50
|
- name: Cache dependencies
|
53
|
-
uses: actions/cache@
|
51
|
+
uses: actions/cache@v3
|
54
52
|
with:
|
55
53
|
path: vendor/bundle
|
56
54
|
key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
|
data/.rubocop.yml
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require:
|
2
|
-
- rubocop-minitest
|
2
|
+
- rubocop-minitest
|
3
|
+
- rubocop/cop/custom
|
3
4
|
|
4
5
|
AllCops:
|
5
6
|
CacheRootDirectory: 'vendor'
|
@@ -14,3 +15,6 @@ Naming/MemoizedInstanceVariableName:
|
|
14
15
|
|
15
16
|
Style/MapToHash:
|
16
17
|
Enabled: false
|
18
|
+
|
19
|
+
Custom/IpRanges:
|
20
|
+
Enabled: true
|
data/README.md
CHANGED
@@ -62,6 +62,7 @@ end
|
|
62
62
|
* [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
|
63
63
|
* [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
|
64
64
|
* [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
|
65
|
+
* [You.com](https://about.you.com/youbot/)
|
65
66
|
|
66
67
|
## License
|
67
68
|
|
data/legitbot.gemspec
CHANGED
@@ -26,11 +26,11 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
|
27
27
|
spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
|
28
28
|
spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
|
29
|
+
spec.add_development_dependency 'nokogiri', '~> 1.13.0', '>= 1.13.0'
|
29
30
|
spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
|
30
|
-
spec.add_development_dependency 'rubocop', '~> 1.
|
31
|
-
spec.add_development_dependency 'rubocop-minitest', '~> 0.
|
31
|
+
spec.add_development_dependency 'rubocop', '~> 1.31.0', '>= 1.31.0'
|
32
|
+
spec.add_development_dependency 'rubocop-minitest', '~> 0.20.0', '>= 0.20.0'
|
32
33
|
|
33
34
|
spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
34
35
|
spec.rdoc_options = ['--charset=UTF-8']
|
35
|
-
spec.test_files = Dir.glob('test/**/*').reject { |f| f.start_with? 'test/lib' }
|
36
36
|
end
|
data/lib/legitbot/duckduckgo.rb
CHANGED
@@ -3,21 +3,21 @@
|
|
3
3
|
module Legitbot # :nodoc:
|
4
4
|
# https://duckduckgo.com/duckduckbot
|
5
5
|
class DuckDuckGo < BotMatch
|
6
|
+
# @fetch:url https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
|
7
|
+
# @fetch:selector section.main article.content ul > li
|
6
8
|
ip_ranges %w[
|
7
|
-
20.191.45.212
|
8
|
-
40.88.21.235
|
9
|
-
40.76.173.151
|
10
|
-
40.76.163.7
|
11
|
-
20.185.79.47
|
12
|
-
52.142.26.175
|
13
9
|
20.185.79.15
|
14
|
-
|
15
|
-
|
16
|
-
40.76.163.23
|
10
|
+
20.185.79.47
|
11
|
+
20.191.45.212
|
17
12
|
40.76.162.191
|
13
|
+
40.76.162.208
|
18
14
|
40.76.162.247
|
19
|
-
|
20
|
-
|
15
|
+
40.76.163.7
|
16
|
+
40.76.163.23
|
17
|
+
40.76.173.151
|
18
|
+
40.88.21.235
|
19
|
+
52.142.24.149
|
20
|
+
52.142.26.175
|
21
21
|
]
|
22
22
|
end
|
23
23
|
|
data/lib/legitbot/pinterest.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Legitbot # :nodoc:
|
4
4
|
# https://help.pinterest.com/en/articles/about-pinterest-crawler-0
|
5
5
|
class Pinterest < BotMatch
|
6
|
-
domains 'pinterest.com.'
|
6
|
+
domains 'pinterest.com.', 'pinterestcrawler.com.'
|
7
7
|
end
|
8
8
|
|
9
9
|
rule Legitbot::Pinterest, %w[Pinterestbot Pinterest/0.2]
|
data/lib/legitbot/version.rb
CHANGED
data/lib/legitbot/you.rb
ADDED
data/lib/legitbot.rb
CHANGED
@@ -13,7 +13,8 @@ require_relative 'legitbot/duckduckgo'
|
|
13
13
|
require_relative 'legitbot/facebook'
|
14
14
|
require_relative 'legitbot/google'
|
15
15
|
require_relative 'legitbot/oracle'
|
16
|
+
require_relative 'legitbot/petalbot'
|
16
17
|
require_relative 'legitbot/pinterest'
|
17
18
|
require_relative 'legitbot/twitter'
|
18
19
|
require_relative 'legitbot/yandex'
|
19
|
-
require_relative 'legitbot/
|
20
|
+
require_relative 'legitbot/you'
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'ipaddr'
|
4
|
+
require 'net/http'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'rubocop'
|
7
|
+
require 'uri'
|
8
|
+
|
9
|
+
module RuboCop
|
10
|
+
module Cop
|
11
|
+
module Custom
|
12
|
+
class IpRanges < Base # :nodoc:
|
13
|
+
extend AutoCorrector
|
14
|
+
|
15
|
+
MSG = 'Outdated list of IP ranges compared to %<url>s'
|
16
|
+
REGEXP = /^\s*#\s*@fetch:(?<param>[a-z0-9_]+)\s+(?<arg>.*)?/
|
17
|
+
|
18
|
+
def_node_matcher :on_ip_ranges, <<~PATTERN
|
19
|
+
(send nil? :ip_ranges $(array str+))
|
20
|
+
PATTERN
|
21
|
+
|
22
|
+
def on_send(node)
|
23
|
+
on_ip_ranges(node) do |value|
|
24
|
+
params = fetch_params(node)
|
25
|
+
return unless mandatory_params?(params)
|
26
|
+
|
27
|
+
existing_ips = read_node_ips value
|
28
|
+
new_ips = fetch_ips(**params)
|
29
|
+
return if existing_ips == new_ips
|
30
|
+
|
31
|
+
register_offense(value, new_ips, **params)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def fetch_ips(url:, selector:)
|
38
|
+
response = Net::HTTP.get_response URI(url)
|
39
|
+
response.value
|
40
|
+
|
41
|
+
document = Nokogiri::HTML response.body
|
42
|
+
document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
|
43
|
+
end
|
44
|
+
|
45
|
+
def read_node_ips(value)
|
46
|
+
value.child_nodes.map(&:value).sort_by(&IPAddr.method(:new))
|
47
|
+
end
|
48
|
+
|
49
|
+
def register_offense(node, new_ips, **params)
|
50
|
+
message = format(MSG, params)
|
51
|
+
add_offense node, message: message do |corrector|
|
52
|
+
corrector.replace node, node_replacement(new_ips)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def mandatory_params?(params)
|
57
|
+
params.include?(:url) && params.include?(:selector)
|
58
|
+
end
|
59
|
+
|
60
|
+
def fetch_params(node)
|
61
|
+
comments = processed_source.ast_with_comments[node]
|
62
|
+
comments.map do |comment|
|
63
|
+
match = comment.text.match(REGEXP)
|
64
|
+
next unless match
|
65
|
+
|
66
|
+
[match[:param].to_sym, match[:arg]]
|
67
|
+
end.compact.to_h
|
68
|
+
end
|
69
|
+
|
70
|
+
def node_replacement(new_ips)
|
71
|
+
contents = new_ips.join("\n")
|
72
|
+
"%w[\n#{contents}\n]"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/test/ahrefs_test.rb
CHANGED
@@ -12,13 +12,13 @@ class AhrefsTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Ahrefs.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '54.36.148.0'
|
20
20
|
match = Legitbot::Ahrefs.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class AhrefsTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,6 +36,6 @@ class AhrefsTest < Minitest::Test
|
|
36
36
|
'54.36.148.0'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
end
|
data/test/alexa_test.rb
CHANGED
@@ -12,13 +12,13 @@ class AlexaTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Alexa.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '52.86.176.3'
|
20
20
|
match = Legitbot::Alexa.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class AlexaTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,6 +36,6 @@ class AlexaTest < Minitest::Test
|
|
36
36
|
'52.86.176.3'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
end
|
data/test/amazon_test.rb
CHANGED
@@ -12,13 +12,13 @@ class AmazonTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Amazon.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '54.166.7.90'
|
20
20
|
match = Legitbot::Amazon.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class AmazonTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,7 +36,7 @@ class AmazonTest < Minitest::Test
|
|
36
36
|
'54.166.7.90'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
|
42
42
|
def test_valid_name
|
data/test/apple_test.rb
CHANGED
@@ -12,13 +12,13 @@ class AppleTest < Minitest::Test
|
|
12
12
|
def test_valid_ip
|
13
13
|
ip = '17.58.98.60'
|
14
14
|
match = Legitbot::Apple.new(ip)
|
15
|
-
|
15
|
+
assert_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_invalid_ip
|
19
19
|
ip = '127.0.0.1'
|
20
20
|
match = Legitbot::Apple.new(ip)
|
21
|
-
|
21
|
+
assert_predicate match, :fake?
|
22
22
|
end
|
23
23
|
|
24
24
|
# rubocop:disable Layout/LineLength
|
@@ -28,7 +28,7 @@ class AppleTest < Minitest::Test
|
|
28
28
|
'17.58.98.60'
|
29
29
|
)
|
30
30
|
assert_equal :apple, bot.detected_as
|
31
|
-
|
31
|
+
assert_predicate bot, :valid?
|
32
32
|
end
|
33
33
|
# rubocop:enable Layout/LineLength
|
34
34
|
end
|
data/test/facebook_test.rb
CHANGED
@@ -20,17 +20,17 @@ class FacebookTest < Minitest::Test
|
|
20
20
|
def test_valid_ip
|
21
21
|
ip = '69.63.186.89'
|
22
22
|
match = Legitbot::Facebook.new(ip)
|
23
|
-
|
23
|
+
assert_predicate match, :valid?
|
24
24
|
|
25
25
|
ip = '69.171.251.1'
|
26
26
|
match = Legitbot::Facebook.new(ip)
|
27
|
-
|
27
|
+
assert_predicate match, :valid?
|
28
28
|
end
|
29
29
|
|
30
30
|
def test_invalid_ip
|
31
31
|
ip = '127.0.0.1'
|
32
32
|
match = Legitbot::Facebook.new(ip)
|
33
|
-
|
33
|
+
assert_predicate match, :fake?
|
34
34
|
end
|
35
35
|
|
36
36
|
def test_user_agent1
|
@@ -39,7 +39,7 @@ class FacebookTest < Minitest::Test
|
|
39
39
|
'31.13.76.56'
|
40
40
|
) do |bot|
|
41
41
|
assert_equal :facebook, bot.detected_as
|
42
|
-
|
42
|
+
assert_predicate bot, :valid?
|
43
43
|
end
|
44
44
|
end
|
45
45
|
|
@@ -49,7 +49,7 @@ class FacebookTest < Minitest::Test
|
|
49
49
|
'173.252.87.8'
|
50
50
|
) do |bot|
|
51
51
|
assert_equal :facebook, bot.detected_as
|
52
|
-
|
52
|
+
assert_predicate bot, :valid?
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
@@ -60,7 +60,7 @@ class FacebookTest < Minitest::Test
|
|
60
60
|
'92.243.181.7'
|
61
61
|
) do |bot|
|
62
62
|
assert_includes %i[facebook twitter], bot.detected_as
|
63
|
-
|
63
|
+
assert_predicate bot, :fake?
|
64
64
|
end
|
65
65
|
end
|
66
66
|
# rubocop:enable Layout/LineLength
|
data/test/google_test.rb
CHANGED
@@ -12,13 +12,13 @@ class GoogleTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Google.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '66.249.64.141'
|
20
20
|
match = Legitbot::Google.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class GoogleTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,7 +36,7 @@ class GoogleTest < Minitest::Test
|
|
36
36
|
'66.249.64.141'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
|
42
42
|
def test_valid_name
|
@@ -87,7 +87,7 @@ module Legitbot
|
|
87
87
|
assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
|
88
88
|
|
89
89
|
matcher = EmptyRanges.new '127.0.0.0'
|
90
|
-
|
90
|
+
assert_predicate matcher, :valid_ip?
|
91
91
|
end
|
92
92
|
|
93
93
|
def test_matcher_array
|
@@ -107,15 +107,15 @@ module Legitbot
|
|
107
107
|
# rubocop:disable Metrics/AbcSize
|
108
108
|
def test_matcher_loader
|
109
109
|
assert_equal 0, LoadRanges.counter
|
110
|
-
|
111
|
-
|
112
|
-
|
110
|
+
assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
111
|
+
refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
|
112
|
+
refute_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
113
113
|
assert_equal 1, LoadRanges.counter
|
114
114
|
|
115
115
|
LoadRanges.reload_ips
|
116
|
-
|
117
|
-
|
118
|
-
|
116
|
+
refute_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
117
|
+
refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
|
118
|
+
assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
119
119
|
assert_equal 2, LoadRanges.counter
|
120
120
|
end
|
121
121
|
# rubocop:enable Metrics/AbcSize
|
data/test/oracle_test.rb
CHANGED
@@ -7,13 +7,13 @@ class OracleTest < Minitest::Test
|
|
7
7
|
def test_malicious_ip
|
8
8
|
ip = '149.210.164.47'
|
9
9
|
match = Legitbot::Oracle.new ip
|
10
|
-
|
10
|
+
refute_predicate match, :valid?
|
11
11
|
end
|
12
12
|
|
13
13
|
def test_valid_ip
|
14
14
|
ip = '148.64.56.64'
|
15
15
|
match = Legitbot::Oracle.new ip
|
16
|
-
|
16
|
+
assert_predicate match, :valid?
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_malicious_ua
|
@@ -22,7 +22,7 @@ class OracleTest < Minitest::Test
|
|
22
22
|
'149.210.164.47'
|
23
23
|
)
|
24
24
|
assert bot
|
25
|
-
|
25
|
+
refute_predicate bot, :valid?
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_valid_ua
|
@@ -31,6 +31,6 @@ class OracleTest < Minitest::Test
|
|
31
31
|
'148.64.56.64'
|
32
32
|
)
|
33
33
|
assert bot
|
34
|
-
|
34
|
+
assert_predicate bot, :valid?
|
35
35
|
end
|
36
36
|
end
|
data/test/petalbot_test.rb
CHANGED
@@ -12,13 +12,13 @@ class PetalbotTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Petalbot.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '114.119.134.10'
|
20
20
|
match = Legitbot::Petalbot.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class PetalbotTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,7 +36,7 @@ class PetalbotTest < Minitest::Test
|
|
36
36
|
'114.119.134.10'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
|
42
42
|
def test_valid_name
|
data/test/pinterest_test.rb
CHANGED
@@ -12,13 +12,13 @@ class PinterestTest < Minitest::Test
|
|
12
12
|
def test_malicious_ip
|
13
13
|
ip = '149.210.164.47'
|
14
14
|
match = Legitbot::Pinterest.new ip
|
15
|
-
|
15
|
+
refute_predicate match, :valid?
|
16
16
|
end
|
17
17
|
|
18
18
|
def test_valid_ip
|
19
19
|
ip = '54.236.1.11'
|
20
20
|
match = Legitbot::Pinterest.new ip
|
21
|
-
|
21
|
+
assert_predicate match, :valid?
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_malicious_ua
|
@@ -27,7 +27,7 @@ class PinterestTest < Minitest::Test
|
|
27
27
|
'149.210.164.47'
|
28
28
|
)
|
29
29
|
assert bot
|
30
|
-
|
30
|
+
refute_predicate bot, :valid?
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_valid_ua
|
@@ -36,7 +36,7 @@ class PinterestTest < Minitest::Test
|
|
36
36
|
'54.236.1.11'
|
37
37
|
)
|
38
38
|
assert bot
|
39
|
-
|
39
|
+
assert_predicate bot, :valid?
|
40
40
|
end
|
41
41
|
|
42
42
|
# rubocop:disable Layout/LineLength
|
data/test/twitter_test.rb
CHANGED
@@ -7,13 +7,13 @@ class TwitterTest < Minitest::Test
|
|
7
7
|
def test_malicious_ip
|
8
8
|
ip = '149.210.164.47'
|
9
9
|
match = Legitbot::Twitter.new ip
|
10
|
-
|
10
|
+
refute_predicate match, :valid?
|
11
11
|
end
|
12
12
|
|
13
13
|
def test_valid_ip
|
14
14
|
ip = '199.16.156.125'
|
15
15
|
match = Legitbot::Twitter.new ip
|
16
|
-
|
16
|
+
assert_predicate match, :valid?
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_malicious_ua
|
@@ -22,7 +22,7 @@ class TwitterTest < Minitest::Test
|
|
22
22
|
'149.210.164.47'
|
23
23
|
)
|
24
24
|
assert bot
|
25
|
-
|
25
|
+
refute_predicate bot, :valid?
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_valid_ua
|
@@ -31,6 +31,6 @@ class TwitterTest < Minitest::Test
|
|
31
31
|
'199.16.156.125'
|
32
32
|
)
|
33
33
|
assert bot
|
34
|
-
|
34
|
+
assert_predicate bot, :valid?
|
35
35
|
end
|
36
36
|
end
|
data/test/you_test.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'minitest/autorun'
|
4
|
+
require 'legitbot'
|
5
|
+
|
6
|
+
class YouTest < Minitest::Test
|
7
|
+
def test_malicious_ip
|
8
|
+
ip = '20.59.41.22'
|
9
|
+
match = Legitbot::You.new ip
|
10
|
+
refute_predicate match, :valid?
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_valid_ip
|
14
|
+
ip = '20.59.40.22'
|
15
|
+
match = Legitbot::You.new ip
|
16
|
+
assert_predicate match, :valid?
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_malicious_ua
|
20
|
+
bot = Legitbot.bot(
|
21
|
+
'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
|
22
|
+
'20.59.41.22'
|
23
|
+
)
|
24
|
+
assert bot
|
25
|
+
refute_predicate bot, :valid?
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_valid_ua
|
29
|
+
bot = Legitbot.bot(
|
30
|
+
'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
|
31
|
+
'20.59.40.22'
|
32
|
+
)
|
33
|
+
assert bot
|
34
|
+
assert_predicate bot, :valid?
|
35
|
+
end
|
36
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: legitbot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Azarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-07-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fast_interval_tree
|
@@ -130,6 +130,26 @@ dependencies:
|
|
130
130
|
- - ">="
|
131
131
|
- !ruby/object:Gem::Version
|
132
132
|
version: 1.5.0
|
133
|
+
- !ruby/object:Gem::Dependency
|
134
|
+
name: nokogiri
|
135
|
+
requirement: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - "~>"
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: 1.13.0
|
140
|
+
- - ">="
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: 1.13.0
|
143
|
+
type: :development
|
144
|
+
prerelease: false
|
145
|
+
version_requirements: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.13.0
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 1.13.0
|
133
153
|
- !ruby/object:Gem::Dependency
|
134
154
|
name: rake
|
135
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,40 +176,40 @@ dependencies:
|
|
156
176
|
requirements:
|
157
177
|
- - "~>"
|
158
178
|
- !ruby/object:Gem::Version
|
159
|
-
version: 1.
|
179
|
+
version: 1.31.0
|
160
180
|
- - ">="
|
161
181
|
- !ruby/object:Gem::Version
|
162
|
-
version: 1.
|
182
|
+
version: 1.31.0
|
163
183
|
type: :development
|
164
184
|
prerelease: false
|
165
185
|
version_requirements: !ruby/object:Gem::Requirement
|
166
186
|
requirements:
|
167
187
|
- - "~>"
|
168
188
|
- !ruby/object:Gem::Version
|
169
|
-
version: 1.
|
189
|
+
version: 1.31.0
|
170
190
|
- - ">="
|
171
191
|
- !ruby/object:Gem::Version
|
172
|
-
version: 1.
|
192
|
+
version: 1.31.0
|
173
193
|
- !ruby/object:Gem::Dependency
|
174
194
|
name: rubocop-minitest
|
175
195
|
requirement: !ruby/object:Gem::Requirement
|
176
196
|
requirements:
|
177
197
|
- - "~>"
|
178
198
|
- !ruby/object:Gem::Version
|
179
|
-
version: 0.
|
199
|
+
version: 0.20.0
|
180
200
|
- - ">="
|
181
201
|
- !ruby/object:Gem::Version
|
182
|
-
version: 0.
|
202
|
+
version: 0.20.0
|
183
203
|
type: :development
|
184
204
|
prerelease: false
|
185
205
|
version_requirements: !ruby/object:Gem::Requirement
|
186
206
|
requirements:
|
187
207
|
- - "~>"
|
188
208
|
- !ruby/object:Gem::Version
|
189
|
-
version: 0.
|
209
|
+
version: 0.20.0
|
190
210
|
- - ">="
|
191
211
|
- !ruby/object:Gem::Version
|
192
|
-
version: 0.
|
212
|
+
version: 0.20.0
|
193
213
|
description: Does Web request come from a real search engine or from an impersonating
|
194
214
|
agent?
|
195
215
|
email: self@alaz.me
|
@@ -198,6 +218,7 @@ extensions: []
|
|
198
218
|
extra_rdoc_files: []
|
199
219
|
files:
|
200
220
|
- ".editorconfig"
|
221
|
+
- ".github/workflows/autocorrect.yml"
|
201
222
|
- ".github/workflows/build.yml"
|
202
223
|
- ".gitignore"
|
203
224
|
- ".rubocop.yml"
|
@@ -228,6 +249,9 @@ files:
|
|
228
249
|
- lib/legitbot/validators/ip_ranges.rb
|
229
250
|
- lib/legitbot/version.rb
|
230
251
|
- lib/legitbot/yandex.rb
|
252
|
+
- lib/legitbot/you.rb
|
253
|
+
- lib/rubocop/cop/custom.rb
|
254
|
+
- lib/rubocop/cop/custom/ip_ranges.rb
|
231
255
|
- rakelib/bump.rake
|
232
256
|
- rakelib/bundler.rake
|
233
257
|
- rakelib/console.rake
|
@@ -247,6 +271,7 @@ files:
|
|
247
271
|
- test/petalbot_test.rb
|
248
272
|
- test/pinterest_test.rb
|
249
273
|
- test/twitter_test.rb
|
274
|
+
- test/you_test.rb
|
250
275
|
homepage: https://github.com/alaz/legitbot
|
251
276
|
licenses:
|
252
277
|
- Apache-2.0
|
@@ -272,18 +297,4 @@ rubygems_version: 3.2.32
|
|
272
297
|
signing_key:
|
273
298
|
specification_version: 4
|
274
299
|
summary: 'Validate requests from Web crawlers: impersonating or not?'
|
275
|
-
test_files:
|
276
|
-
- test/ahrefs_test.rb
|
277
|
-
- test/alexa_test.rb
|
278
|
-
- test/amazon_test.rb
|
279
|
-
- test/apple_test.rb
|
280
|
-
- test/botmatch_test.rb
|
281
|
-
- test/facebook_test.rb
|
282
|
-
- test/google_test.rb
|
283
|
-
- test/legitbot/validators/domains_test.rb
|
284
|
-
- test/legitbot/validators/ip_ranges_test.rb
|
285
|
-
- test/legitbot_test.rb
|
286
|
-
- test/oracle_test.rb
|
287
|
-
- test/petalbot_test.rb
|
288
|
-
- test/pinterest_test.rb
|
289
|
-
- test/twitter_test.rb
|
300
|
+
test_files: []
|