legitbot 1.5.0 → 1.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/autocorrect.yml +44 -0
- data/.github/workflows/build.yml +2 -4
- data/.rubocop.yml +10 -0
- data/.ruby-version +1 -1
- data/README.md +3 -1
- data/Rakefile +0 -24
- data/legitbot.gemspec +13 -4
- data/lib/legitbot/duckduckgo.rb +12 -11
- data/lib/legitbot/validators/domains.rb +2 -0
- data/lib/legitbot/version.rb +1 -1
- data/lib/rubocop/cop/custom/ip_ranges.rb +77 -0
- data/lib/rubocop/cop/custom.rb +3 -0
- data/rakelib/bump.rake +5 -0
- data/rakelib/bundler.rake +4 -0
- data/rakelib/console.rake +8 -0
- data/rakelib/test.rake +12 -0
- data/test/ahrefs_test.rb +11 -6
- data/test/alexa_test.rb +11 -6
- data/test/amazon_test.rb +11 -6
- data/test/apple_test.rb +8 -3
- data/test/botmatch_test.rb +7 -2
- data/test/facebook_test.rb +14 -10
- data/test/google_test.rb +11 -6
- data/test/legitbot/validators/ip_ranges_test.rb +9 -7
- data/test/legitbot_test.rb +2 -4
- data/test/lib/dns_server_mock.rb +89 -0
- data/test/oracle_test.rb +6 -6
- data/test/petalbot_test.rb +13 -8
- data/test/pinterest_test.rb +11 -6
- data/test/twitter_test.rb +6 -6
- metadata +112 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0bd071c617ddb67424715ae30b92783fa0155ca182c7ef0c7cac0a5962eff8d1
|
4
|
+
data.tar.gz: 33ed371772200f2d27d95d083782d73d1572eeac329f63e5e3d5f14937151b72
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d188e8a97d9fd5115897f69c31a2f43f8b55b0426bd932c347800fd023231dfbc1c6c4ada108c94cdd4258673cbee06d012182ef3e815ff7be8b010b9d4eb4db
|
7
|
+
data.tar.gz: a00e24aa78775c201f9587286bb7cec46ec9993687ace3e675c2e4f585ff4ff56839f9a977e9709dab13e652098e72910a6f86b4dbc5fa38083970ca9085c143
|
@@ -0,0 +1,44 @@
|
|
1
|
+
name: autocorrect
|
2
|
+
|
3
|
+
on:
|
4
|
+
schedule:
|
5
|
+
- cron: '29 6 * * 6'
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
lint:
|
9
|
+
runs-on: ubuntu-latest
|
10
|
+
|
11
|
+
strategy:
|
12
|
+
matrix:
|
13
|
+
ruby: [ 3.0 ]
|
14
|
+
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- name: Set up Ruby
|
18
|
+
uses: ruby/setup-ruby@v1
|
19
|
+
with:
|
20
|
+
ruby-version: ${{ matrix.ruby }}
|
21
|
+
- name: Cache dependencies
|
22
|
+
uses: actions/cache@v1
|
23
|
+
with:
|
24
|
+
path: vendor/bundle
|
25
|
+
key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
|
26
|
+
restore-keys: |
|
27
|
+
${{ runner.os }}-${{ matrix.ruby }}-gems-
|
28
|
+
- name: Install dependencies
|
29
|
+
run: |
|
30
|
+
bundle config path vendor/bundle
|
31
|
+
bundle install --jobs 4 --retry 3
|
32
|
+
- name: Run tests
|
33
|
+
run: bundle exec rake test
|
34
|
+
- name: Run linter
|
35
|
+
run: bundle exec rubocop --auto-correct
|
36
|
+
- name: Create Pull Request
|
37
|
+
uses: peter-evans/create-pull-request@v3
|
38
|
+
with:
|
39
|
+
branch: update/lint-autocorrect
|
40
|
+
delete-branch: true
|
41
|
+
commit-message: '🔧 autocorrect by Rubocop'
|
42
|
+
title: '[rubocop] Automatic corrections'
|
43
|
+
body: |
|
44
|
+
Please review automatic changes proposed by `rubocop -a`.
|
data/.github/workflows/build.yml
CHANGED
@@ -3,8 +3,6 @@ name: build
|
|
3
3
|
on:
|
4
4
|
pull_request:
|
5
5
|
push:
|
6
|
-
schedule:
|
7
|
-
- cron: '29 6 * * 6'
|
8
6
|
|
9
7
|
jobs:
|
10
8
|
test:
|
@@ -13,7 +11,7 @@ jobs:
|
|
13
11
|
strategy:
|
14
12
|
fail-fast: false
|
15
13
|
matrix:
|
16
|
-
ruby: [ jruby, 2.
|
14
|
+
ruby: [ jruby, 2.6, 2.7, 3.0 ]
|
17
15
|
|
18
16
|
steps:
|
19
17
|
- uses: actions/checkout@v2
|
@@ -41,7 +39,7 @@ jobs:
|
|
41
39
|
|
42
40
|
strategy:
|
43
41
|
matrix:
|
44
|
-
ruby: [
|
42
|
+
ruby: [ 3.0 ]
|
45
43
|
|
46
44
|
steps:
|
47
45
|
- uses: actions/checkout@v2
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
require:
|
2
|
+
- rubocop-minitest
|
3
|
+
- rubocop/cop/custom
|
4
|
+
|
1
5
|
AllCops:
|
2
6
|
CacheRootDirectory: 'vendor'
|
3
7
|
NewCops: enable
|
@@ -8,3 +12,9 @@ Gemspec/RequiredRubyVersion:
|
|
8
12
|
|
9
13
|
Naming/MemoizedInstanceVariableName:
|
10
14
|
Enabled: false
|
15
|
+
|
16
|
+
Style/MapToHash:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Custom/IpRanges:
|
20
|
+
Enabled: true
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
3.0.3
|
data/README.md
CHANGED
@@ -67,7 +67,7 @@ end
|
|
67
67
|
|
68
68
|
Apache 2.0
|
69
69
|
|
70
|
-
##
|
70
|
+
## Other projects
|
71
71
|
|
72
72
|
* Play Framework variant in Scala: [play-legitbot](https://github.com/osinka/play-legitbot)
|
73
73
|
* Article [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
|
@@ -80,3 +80,5 @@ Apache 2.0
|
|
80
80
|
classify IP as a search engine, but also label them as suspicious and
|
81
81
|
reports the number of days since the last activity. My implementation of
|
82
82
|
the protocol in Scala is [here](https://github.com/osinka/httpbl).
|
83
|
+
* [CIDRAM](https://github.com/CIDRAM/CIDRAM) is a PHP routing manager with built-in support
|
84
|
+
to validate bots.
|
data/Rakefile
CHANGED
@@ -1,25 +1 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'bundler'
|
5
|
-
require 'bump/tasks'
|
6
|
-
require 'rake/testtask'
|
7
|
-
Bundler::GemHelper.install_tasks
|
8
|
-
|
9
|
-
Bump.tag_by_default = true
|
10
|
-
|
11
|
-
Rake::TestTask.new do |t|
|
12
|
-
t.libs << 'test'
|
13
|
-
t.test_files = FileList['test/**/*_test.rb']
|
14
|
-
t.warning = true
|
15
|
-
t.verbose = true
|
16
|
-
end
|
17
|
-
|
18
|
-
desc 'Start a console'
|
19
|
-
task :console do
|
20
|
-
require 'irb'
|
21
|
-
ARGV.clear
|
22
|
-
IRB.start
|
23
|
-
end
|
24
|
-
|
25
|
-
task default: %w[test]
|
data/legitbot.gemspec
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'English'
|
4
|
+
|
3
5
|
$LOAD_PATH.push File.expand_path('lib', __dir__)
|
4
6
|
require 'legitbot/version'
|
5
7
|
|
@@ -13,16 +15,23 @@ Gem::Specification.new do |spec|
|
|
13
15
|
spec.homepage = 'https://github.com/alaz/legitbot'
|
14
16
|
spec.summary = 'Validate requests from Web crawlers: impersonating or not?'
|
15
17
|
spec.description = 'Does Web request come from a real search engine or from an impersonating agent?'
|
18
|
+
spec.metadata = {
|
19
|
+
'rubygems_mfa_required' => 'true'
|
20
|
+
}
|
16
21
|
|
17
|
-
spec.required_ruby_version = '>= 2.
|
18
|
-
spec.add_dependency '
|
22
|
+
spec.required_ruby_version = '>= 2.6.0'
|
23
|
+
spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
|
19
24
|
spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
|
20
25
|
spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
|
26
|
+
spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
|
21
27
|
spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
|
28
|
+
spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
|
29
|
+
spec.add_development_dependency 'nokogiri', '~> 1.13.0', '>= 1.13.0'
|
22
30
|
spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
|
23
|
-
spec.add_development_dependency 'rubocop', '~> 1.
|
31
|
+
spec.add_development_dependency 'rubocop', '~> 1.28.0', '>= 1.28.0'
|
32
|
+
spec.add_development_dependency 'rubocop-minitest', '~> 0.19.0', '>= 0.19.0'
|
24
33
|
|
25
34
|
spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
26
35
|
spec.rdoc_options = ['--charset=UTF-8']
|
27
|
-
spec.test_files = Dir.glob('test/**/*')
|
36
|
+
spec.test_files = Dir.glob('test/**/*').reject { |f| f.start_with? 'test/lib' }
|
28
37
|
end
|
data/lib/legitbot/duckduckgo.rb
CHANGED
@@ -3,20 +3,21 @@
|
|
3
3
|
module Legitbot # :nodoc:
|
4
4
|
# https://duckduckgo.com/duckduckbot
|
5
5
|
class DuckDuckGo < BotMatch
|
6
|
+
# @fetch:url https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
|
7
|
+
# @fetch:selector section.main article.content ul > li
|
6
8
|
ip_ranges %w[
|
9
|
+
20.185.79.15
|
10
|
+
20.185.79.47
|
7
11
|
20.191.45.212
|
8
|
-
|
12
|
+
40.76.162.191
|
13
|
+
40.76.162.208
|
14
|
+
40.76.162.247
|
15
|
+
40.76.163.7
|
16
|
+
40.76.163.23
|
17
|
+
40.76.173.151
|
9
18
|
40.88.21.235
|
10
|
-
|
11
|
-
|
12
|
-
50.16.241.117
|
13
|
-
50.16.247.234
|
14
|
-
52.5.190.19
|
15
|
-
52.204.97.54
|
16
|
-
54.197.234.188
|
17
|
-
54.208.100.253
|
18
|
-
54.208.102.37
|
19
|
-
107.21.1.8
|
19
|
+
52.142.24.149
|
20
|
+
52.142.26.175
|
20
21
|
]
|
21
22
|
end
|
22
23
|
|
data/lib/legitbot/version.rb
CHANGED
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'ipaddr'
|
4
|
+
require 'net/http'
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'rubocop'
|
7
|
+
require 'uri'
|
8
|
+
|
9
|
+
module RuboCop
|
10
|
+
module Cop
|
11
|
+
module Custom
|
12
|
+
class IpRanges < Base # :nodoc:
|
13
|
+
extend AutoCorrector
|
14
|
+
|
15
|
+
MSG = 'Outdated list of IP ranges compared to %<url>s'
|
16
|
+
REGEXP = /^\s*#\s*@fetch:(?<param>[a-z0-9_]+)\s+(?<arg>.*)?/
|
17
|
+
|
18
|
+
def_node_matcher :on_ip_ranges, <<~PATTERN
|
19
|
+
(send nil? :ip_ranges $(array str+))
|
20
|
+
PATTERN
|
21
|
+
|
22
|
+
def on_send(node)
|
23
|
+
on_ip_ranges(node) do |value|
|
24
|
+
params = fetch_params(node)
|
25
|
+
return unless mandatory_params?(params)
|
26
|
+
|
27
|
+
existing_ips = read_node_ips value
|
28
|
+
new_ips = fetch_ips(**params)
|
29
|
+
return if existing_ips == new_ips
|
30
|
+
|
31
|
+
register_offense(value, new_ips, **params)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def fetch_ips(url:, selector:)
|
38
|
+
response = Net::HTTP.get_response URI(url)
|
39
|
+
response.value
|
40
|
+
|
41
|
+
document = Nokogiri::HTML response.body
|
42
|
+
document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
|
43
|
+
end
|
44
|
+
|
45
|
+
def read_node_ips(value)
|
46
|
+
value.child_nodes.map(&:value).sort_by(&IPAddr.method(:new))
|
47
|
+
end
|
48
|
+
|
49
|
+
def register_offense(node, new_ips, **params)
|
50
|
+
message = format(MSG, params)
|
51
|
+
add_offense node, message: message do |corrector|
|
52
|
+
corrector.replace node, node_replacement(new_ips)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def mandatory_params?(params)
|
57
|
+
params.include?(:url) && params.include?(:selector)
|
58
|
+
end
|
59
|
+
|
60
|
+
def fetch_params(node)
|
61
|
+
comments = processed_source.ast_with_comments[node]
|
62
|
+
comments.map do |comment|
|
63
|
+
match = comment.text.match(REGEXP)
|
64
|
+
next unless match
|
65
|
+
|
66
|
+
[match[:param].to_sym, match[:arg]]
|
67
|
+
end.compact.to_h
|
68
|
+
end
|
69
|
+
|
70
|
+
def node_replacement(new_ips)
|
71
|
+
contents = new_ips.join("\n")
|
72
|
+
"%w[\n#{contents}\n]"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/rakelib/bump.rake
ADDED
data/rakelib/test.rake
ADDED
data/test/ahrefs_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class AhrefsTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Ahrefs.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
19
|
ip = '54.36.148.0'
|
15
20
|
match = Legitbot::Ahrefs.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,8 +26,8 @@ class AhrefsTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
@@ -30,7 +35,7 @@ class AhrefsTest < Minitest::Test
|
|
30
35
|
'Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)',
|
31
36
|
'54.36.148.0'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
end
|
data/test/alexa_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class AlexaTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Alexa.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
19
|
ip = '52.86.176.3'
|
15
20
|
match = Legitbot::Alexa.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,8 +26,8 @@ class AlexaTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
@@ -30,7 +35,7 @@ class AlexaTest < Minitest::Test
|
|
30
35
|
'Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)',
|
31
36
|
'52.86.176.3'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
end
|
data/test/amazon_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class AmazonTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Amazon.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
19
|
ip = '54.166.7.90'
|
15
20
|
match = Legitbot::Amazon.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,8 +26,8 @@ class AmazonTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
@@ -30,8 +35,8 @@ class AmazonTest < Minitest::Test
|
|
30
35
|
'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
|
31
36
|
'54.166.7.90'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
|
37
42
|
def test_valid_name
|
data/test/apple_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class AppleTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_valid_ip
|
8
13
|
ip = '17.58.98.60'
|
9
14
|
match = Legitbot::Apple.new(ip)
|
10
|
-
|
15
|
+
assert_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_invalid_ip
|
14
19
|
ip = '127.0.0.1'
|
15
20
|
match = Legitbot::Apple.new(ip)
|
16
|
-
|
21
|
+
assert_predicate match, :fake?
|
17
22
|
end
|
18
23
|
|
19
24
|
# rubocop:disable Layout/LineLength
|
@@ -23,7 +28,7 @@ class AppleTest < Minitest::Test
|
|
23
28
|
'17.58.98.60'
|
24
29
|
)
|
25
30
|
assert_equal :apple, bot.detected_as
|
26
|
-
|
31
|
+
assert_predicate bot, :valid?
|
27
32
|
end
|
28
33
|
# rubocop:enable Layout/LineLength
|
29
34
|
end
|
data/test/botmatch_test.rb
CHANGED
@@ -1,11 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class BotMatchTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_valid_class_syntax
|
8
|
-
assert Legitbot::Google.valid?('66.249.64.141')
|
9
|
-
assert Legitbot::Google.fake?('149.210.164.47')
|
13
|
+
assert Legitbot::Google.valid?('66.249.64.141')
|
14
|
+
assert Legitbot::Google.fake?('149.210.164.47')
|
10
15
|
end
|
11
16
|
end
|
data/test/facebook_test.rb
CHANGED
@@ -20,44 +20,48 @@ class FacebookTest < Minitest::Test
|
|
20
20
|
def test_valid_ip
|
21
21
|
ip = '69.63.186.89'
|
22
22
|
match = Legitbot::Facebook.new(ip)
|
23
|
-
|
23
|
+
assert_predicate match, :valid?
|
24
24
|
|
25
25
|
ip = '69.171.251.1'
|
26
26
|
match = Legitbot::Facebook.new(ip)
|
27
|
-
|
27
|
+
assert_predicate match, :valid?
|
28
28
|
end
|
29
29
|
|
30
30
|
def test_invalid_ip
|
31
31
|
ip = '127.0.0.1'
|
32
32
|
match = Legitbot::Facebook.new(ip)
|
33
|
-
|
33
|
+
assert_predicate match, :fake?
|
34
34
|
end
|
35
35
|
|
36
|
-
|
37
|
-
def test_user_agent
|
36
|
+
def test_user_agent1
|
38
37
|
Legitbot.bot(
|
39
38
|
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
|
40
39
|
'31.13.76.56'
|
41
40
|
) do |bot|
|
42
41
|
assert_equal :facebook, bot.detected_as
|
43
|
-
|
42
|
+
assert_predicate bot, :valid?
|
44
43
|
end
|
44
|
+
end
|
45
45
|
|
46
|
+
def test_user_agent2
|
46
47
|
Legitbot.bot(
|
47
48
|
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
|
48
49
|
'173.252.87.8'
|
49
50
|
) do |bot|
|
50
51
|
assert_equal :facebook, bot.detected_as
|
51
|
-
|
52
|
+
assert_predicate bot, :valid?
|
52
53
|
end
|
54
|
+
end
|
53
55
|
|
56
|
+
# rubocop:disable Layout/LineLength
|
57
|
+
def test_user_agent3
|
54
58
|
Legitbot.bot(
|
55
59
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
|
56
60
|
'92.243.181.7'
|
57
61
|
) do |bot|
|
58
|
-
|
59
|
-
|
62
|
+
assert_includes %i[facebook twitter], bot.detected_as
|
63
|
+
assert_predicate bot, :fake?
|
60
64
|
end
|
61
65
|
end
|
62
|
-
# rubocop:enable Layout/LineLength
|
66
|
+
# rubocop:enable Layout/LineLength
|
63
67
|
end
|
data/test/google_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class GoogleTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Google.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
19
|
ip = '66.249.64.141'
|
15
20
|
match = Legitbot::Google.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,8 +26,8 @@ class GoogleTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
@@ -30,8 +35,8 @@ class GoogleTest < Minitest::Test
|
|
30
35
|
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
31
36
|
'66.249.64.141'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
|
37
42
|
def test_valid_name
|
@@ -4,6 +4,7 @@ require 'minitest/autorun'
|
|
4
4
|
require 'legitbot'
|
5
5
|
|
6
6
|
module Legitbot
|
7
|
+
# rubocop:disable Minitest/MultipleAssertions
|
7
8
|
module Validators
|
8
9
|
class NoRanges
|
9
10
|
include IpRanges
|
@@ -86,7 +87,7 @@ module Legitbot
|
|
86
87
|
assert EmptyRanges.valid_ip?('2a03:2880:f234:0:0:0:0:1')
|
87
88
|
|
88
89
|
matcher = EmptyRanges.new '127.0.0.0'
|
89
|
-
|
90
|
+
assert_predicate matcher, :valid_ip?
|
90
91
|
end
|
91
92
|
|
92
93
|
def test_matcher_array
|
@@ -106,15 +107,15 @@ module Legitbot
|
|
106
107
|
# rubocop:disable Metrics/AbcSize
|
107
108
|
def test_matcher_loader
|
108
109
|
assert_equal 0, LoadRanges.counter
|
109
|
-
|
110
|
-
|
111
|
-
|
110
|
+
assert_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
111
|
+
refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
|
112
|
+
refute_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
112
113
|
assert_equal 1, LoadRanges.counter
|
113
114
|
|
114
115
|
LoadRanges.reload_ips
|
115
|
-
|
116
|
-
|
117
|
-
|
116
|
+
refute_predicate LoadRanges.new('127.127.127.127'), :valid_ip?
|
117
|
+
refute_predicate LoadRanges.new('10.10.10.10'), :valid_ip?
|
118
|
+
assert_predicate LoadRanges.new('192.168.127.254'), :valid_ip?
|
118
119
|
assert_equal 2, LoadRanges.counter
|
119
120
|
end
|
120
121
|
# rubocop:enable Metrics/AbcSize
|
@@ -128,4 +129,5 @@ module Legitbot
|
|
128
129
|
end
|
129
130
|
end
|
130
131
|
end
|
132
|
+
# rubocop:enable Minitest/MultipleAssertions
|
131
133
|
end
|
data/test/legitbot_test.rb
CHANGED
@@ -5,10 +5,8 @@ require 'legitbot'
|
|
5
5
|
|
6
6
|
class LegitbotTest < Minitest::Test
|
7
7
|
def test_rules
|
8
|
-
|
9
|
-
|
10
|
-
assert Legitbot.bot('Googlebot', '5.140.70.64'),
|
11
|
-
msg: 'No reverse resolve, bot'
|
8
|
+
refute Legitbot.bot('Firefox', '127.0.0.1')
|
9
|
+
assert Legitbot.bot('Googlebot', '5.140.70.64')
|
12
10
|
|
13
11
|
Legitbot.bot('Firefox', '127.0.0.1') do |_bot|
|
14
12
|
flunk 'No bot Firefox is possible'
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'dns_mock'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
TEST_DNS_RECORDS = {
|
7
|
+
# Malicious
|
8
|
+
'149.210.164.47' => {
|
9
|
+
ptr: %w[malicious.spam.co]
|
10
|
+
},
|
11
|
+
|
12
|
+
# Ahrefs
|
13
|
+
'ip-54-36-148-0.a.ahrefs.com' => {
|
14
|
+
a: %w[54.36.148.0]
|
15
|
+
},
|
16
|
+
'54.36.148.0' => {
|
17
|
+
ptr: %w[ip-54-36-148-0.a.ahrefs.com]
|
18
|
+
},
|
19
|
+
|
20
|
+
# Alexa
|
21
|
+
'52.86.176.3' => {
|
22
|
+
ptr: %w[crawl-52-86-176-3.alexa.com]
|
23
|
+
},
|
24
|
+
|
25
|
+
# Amazon
|
26
|
+
'crawler-54-166-7-90.amazonadbot.com' => {
|
27
|
+
a: %w[54.166.7.90]
|
28
|
+
},
|
29
|
+
'54.166.7.90' => {
|
30
|
+
ptr: %w[crawler-54-166-7-90.amazonadbot.com]
|
31
|
+
},
|
32
|
+
|
33
|
+
# Apple
|
34
|
+
'17-58-98-60.applebot.apple.com' => {
|
35
|
+
a: %w[17.58.98.60]
|
36
|
+
},
|
37
|
+
'17.58.98.60' => {
|
38
|
+
ptr: %w[17-58-98-60.applebot.apple.com]
|
39
|
+
},
|
40
|
+
|
41
|
+
# Google
|
42
|
+
'crawl-66-249-64-141.googlebot.com' => {
|
43
|
+
a: %w[66.249.64.141]
|
44
|
+
},
|
45
|
+
'66.249.64.141' => {
|
46
|
+
ptr: %w[crawl-66-249-64-141.googlebot.com]
|
47
|
+
},
|
48
|
+
|
49
|
+
# Petalbot
|
50
|
+
'petalbot-114-119-134-10.petalsearch.com' => {
|
51
|
+
a: %w[114.119.134.10]
|
52
|
+
},
|
53
|
+
'114.119.134.10' => {
|
54
|
+
ptr: %w[petalbot-114-119-134-10.petalsearch.com]
|
55
|
+
},
|
56
|
+
|
57
|
+
# Pinterest
|
58
|
+
'crawl-54-236-1-11.pinterest.com' => {
|
59
|
+
a: %w[54.236.1.11]
|
60
|
+
},
|
61
|
+
'54.236.1.11' => {
|
62
|
+
ptr: %w[crawl-54-236-1-11.pinterest.com]
|
63
|
+
}
|
64
|
+
}.freeze
|
65
|
+
|
66
|
+
class DnsServer
|
67
|
+
class << self
|
68
|
+
attr_accessor :mock
|
69
|
+
end
|
70
|
+
|
71
|
+
@mock = DnsMock.start_server records: TEST_DNS_RECORDS
|
72
|
+
end
|
73
|
+
|
74
|
+
module DnsServerMock
|
75
|
+
def before_all
|
76
|
+
super
|
77
|
+
|
78
|
+
Legitbot.resolver_config = {
|
79
|
+
nameserver: 'localhost',
|
80
|
+
nameserver_port: [['localhost', DnsServer.mock.port]]
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
def after_all
|
85
|
+
Legitbot.resolver_config = nil
|
86
|
+
|
87
|
+
super
|
88
|
+
end
|
89
|
+
end
|
data/test/oracle_test.rb
CHANGED
@@ -7,13 +7,13 @@ class OracleTest < Minitest::Test
|
|
7
7
|
def test_malicious_ip
|
8
8
|
ip = '149.210.164.47'
|
9
9
|
match = Legitbot::Oracle.new ip
|
10
|
-
|
10
|
+
refute_predicate match, :valid?
|
11
11
|
end
|
12
12
|
|
13
13
|
def test_valid_ip
|
14
14
|
ip = '148.64.56.64'
|
15
15
|
match = Legitbot::Oracle.new ip
|
16
|
-
|
16
|
+
assert_predicate match, :valid?
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_malicious_ua
|
@@ -21,8 +21,8 @@ class OracleTest < Minitest::Test
|
|
21
21
|
'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
|
22
22
|
'149.210.164.47'
|
23
23
|
)
|
24
|
-
assert bot
|
25
|
-
|
24
|
+
assert bot
|
25
|
+
refute_predicate bot, :valid?
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_valid_ua
|
@@ -30,7 +30,7 @@ class OracleTest < Minitest::Test
|
|
30
30
|
'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
|
31
31
|
'148.64.56.64'
|
32
32
|
)
|
33
|
-
assert bot
|
34
|
-
|
33
|
+
assert bot
|
34
|
+
assert_predicate bot, :valid?
|
35
35
|
end
|
36
36
|
end
|
data/test/petalbot_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class PetalbotTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Petalbot.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
|
-
ip = '114.119.
|
19
|
+
ip = '114.119.134.10'
|
15
20
|
match = Legitbot::Petalbot.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,17 +26,17 @@ class PetalbotTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
29
34
|
bot = Legitbot.bot(
|
30
35
|
'Mozilla/5.0 (compatible;PetalBot; +https://aspiegel.com/petalbot)',
|
31
|
-
'114.119.
|
36
|
+
'114.119.134.10'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
|
37
42
|
def test_valid_name
|
data/test/pinterest_test.rb
CHANGED
@@ -1,19 +1,24 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'minitest/autorun'
|
4
|
+
require 'minitest/hooks/test'
|
5
|
+
require 'lib/dns_server_mock'
|
4
6
|
require 'legitbot'
|
5
7
|
|
6
8
|
class PinterestTest < Minitest::Test
|
9
|
+
include Minitest::Hooks
|
10
|
+
include DnsServerMock
|
11
|
+
|
7
12
|
def test_malicious_ip
|
8
13
|
ip = '149.210.164.47'
|
9
14
|
match = Legitbot::Pinterest.new ip
|
10
|
-
|
15
|
+
refute_predicate match, :valid?
|
11
16
|
end
|
12
17
|
|
13
18
|
def test_valid_ip
|
14
19
|
ip = '54.236.1.11'
|
15
20
|
match = Legitbot::Pinterest.new ip
|
16
|
-
|
21
|
+
assert_predicate match, :valid?
|
17
22
|
end
|
18
23
|
|
19
24
|
def test_malicious_ua
|
@@ -21,8 +26,8 @@ class PinterestTest < Minitest::Test
|
|
21
26
|
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
|
22
27
|
'149.210.164.47'
|
23
28
|
)
|
24
|
-
assert bot
|
25
|
-
|
29
|
+
assert bot
|
30
|
+
refute_predicate bot, :valid?
|
26
31
|
end
|
27
32
|
|
28
33
|
def test_valid_ua
|
@@ -30,8 +35,8 @@ class PinterestTest < Minitest::Test
|
|
30
35
|
'Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)',
|
31
36
|
'54.236.1.11'
|
32
37
|
)
|
33
|
-
assert bot
|
34
|
-
|
38
|
+
assert bot
|
39
|
+
assert_predicate bot, :valid?
|
35
40
|
end
|
36
41
|
|
37
42
|
# rubocop:disable Layout/LineLength
|
data/test/twitter_test.rb
CHANGED
@@ -7,13 +7,13 @@ class TwitterTest < Minitest::Test
|
|
7
7
|
def test_malicious_ip
|
8
8
|
ip = '149.210.164.47'
|
9
9
|
match = Legitbot::Twitter.new ip
|
10
|
-
|
10
|
+
refute_predicate match, :valid?
|
11
11
|
end
|
12
12
|
|
13
13
|
def test_valid_ip
|
14
14
|
ip = '199.16.156.125'
|
15
15
|
match = Legitbot::Twitter.new ip
|
16
|
-
|
16
|
+
assert_predicate match, :valid?
|
17
17
|
end
|
18
18
|
|
19
19
|
def test_malicious_ua
|
@@ -21,8 +21,8 @@ class TwitterTest < Minitest::Test
|
|
21
21
|
'Twitterbot/1.0',
|
22
22
|
'149.210.164.47'
|
23
23
|
)
|
24
|
-
assert bot
|
25
|
-
|
24
|
+
assert bot
|
25
|
+
refute_predicate bot, :valid?
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_valid_ua
|
@@ -30,7 +30,7 @@ class TwitterTest < Minitest::Test
|
|
30
30
|
'Twitterbot/1.0',
|
31
31
|
'199.16.156.125'
|
32
32
|
)
|
33
|
-
assert bot
|
34
|
-
|
33
|
+
assert bot
|
34
|
+
assert_predicate bot, :valid?
|
35
35
|
end
|
36
36
|
end
|
metadata
CHANGED
@@ -1,35 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: legitbot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Azarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: fast_interval_tree
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0.
|
19
|
+
version: '0.2'
|
20
20
|
- - ">="
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version: 0.
|
22
|
+
version: 0.2.2
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - "~>"
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '0.
|
29
|
+
version: '0.2'
|
30
30
|
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: 0.
|
32
|
+
version: 0.2.2
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: irrc
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -70,6 +70,26 @@ dependencies:
|
|
70
70
|
- - ">="
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 0.8.0
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: dns_mock
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - "~>"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: 1.5.0
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.5.0
|
83
|
+
type: :development
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 1.5.0
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: 1.5.0
|
73
93
|
- !ruby/object:Gem::Dependency
|
74
94
|
name: minitest
|
75
95
|
requirement: !ruby/object:Gem::Requirement
|
@@ -90,6 +110,46 @@ dependencies:
|
|
90
110
|
- - ">="
|
91
111
|
- !ruby/object:Gem::Version
|
92
112
|
version: 5.1.0
|
113
|
+
- !ruby/object:Gem::Dependency
|
114
|
+
name: minitest-hooks
|
115
|
+
requirement: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - "~>"
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '1.5'
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: 1.5.0
|
123
|
+
type: :development
|
124
|
+
prerelease: false
|
125
|
+
version_requirements: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - "~>"
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '1.5'
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: 1.5.0
|
133
|
+
- !ruby/object:Gem::Dependency
|
134
|
+
name: nokogiri
|
135
|
+
requirement: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - "~>"
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: 1.13.0
|
140
|
+
- - ">="
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: 1.13.0
|
143
|
+
type: :development
|
144
|
+
prerelease: false
|
145
|
+
version_requirements: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.13.0
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 1.13.0
|
93
153
|
- !ruby/object:Gem::Dependency
|
94
154
|
name: rake
|
95
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -116,20 +176,40 @@ dependencies:
|
|
116
176
|
requirements:
|
117
177
|
- - "~>"
|
118
178
|
- !ruby/object:Gem::Version
|
119
|
-
version: 1.
|
179
|
+
version: 1.28.0
|
180
|
+
- - ">="
|
181
|
+
- !ruby/object:Gem::Version
|
182
|
+
version: 1.28.0
|
183
|
+
type: :development
|
184
|
+
prerelease: false
|
185
|
+
version_requirements: !ruby/object:Gem::Requirement
|
186
|
+
requirements:
|
187
|
+
- - "~>"
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: 1.28.0
|
190
|
+
- - ">="
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: 1.28.0
|
193
|
+
- !ruby/object:Gem::Dependency
|
194
|
+
name: rubocop-minitest
|
195
|
+
requirement: !ruby/object:Gem::Requirement
|
196
|
+
requirements:
|
197
|
+
- - "~>"
|
198
|
+
- !ruby/object:Gem::Version
|
199
|
+
version: 0.19.0
|
120
200
|
- - ">="
|
121
201
|
- !ruby/object:Gem::Version
|
122
|
-
version:
|
202
|
+
version: 0.19.0
|
123
203
|
type: :development
|
124
204
|
prerelease: false
|
125
205
|
version_requirements: !ruby/object:Gem::Requirement
|
126
206
|
requirements:
|
127
207
|
- - "~>"
|
128
208
|
- !ruby/object:Gem::Version
|
129
|
-
version:
|
209
|
+
version: 0.19.0
|
130
210
|
- - ">="
|
131
211
|
- !ruby/object:Gem::Version
|
132
|
-
version:
|
212
|
+
version: 0.19.0
|
133
213
|
description: Does Web request come from a real search engine or from an impersonating
|
134
214
|
agent?
|
135
215
|
email: self@alaz.me
|
@@ -138,6 +218,7 @@ extensions: []
|
|
138
218
|
extra_rdoc_files: []
|
139
219
|
files:
|
140
220
|
- ".editorconfig"
|
221
|
+
- ".github/workflows/autocorrect.yml"
|
141
222
|
- ".github/workflows/build.yml"
|
142
223
|
- ".gitignore"
|
143
224
|
- ".rubocop.yml"
|
@@ -168,6 +249,12 @@ files:
|
|
168
249
|
- lib/legitbot/validators/ip_ranges.rb
|
169
250
|
- lib/legitbot/version.rb
|
170
251
|
- lib/legitbot/yandex.rb
|
252
|
+
- lib/rubocop/cop/custom.rb
|
253
|
+
- lib/rubocop/cop/custom/ip_ranges.rb
|
254
|
+
- rakelib/bump.rake
|
255
|
+
- rakelib/bundler.rake
|
256
|
+
- rakelib/console.rake
|
257
|
+
- rakelib/test.rake
|
171
258
|
- test/ahrefs_test.rb
|
172
259
|
- test/alexa_test.rb
|
173
260
|
- test/amazon_test.rb
|
@@ -178,6 +265,7 @@ files:
|
|
178
265
|
- test/legitbot/validators/domains_test.rb
|
179
266
|
- test/legitbot/validators/ip_ranges_test.rb
|
180
267
|
- test/legitbot_test.rb
|
268
|
+
- test/lib/dns_server_mock.rb
|
181
269
|
- test/oracle_test.rb
|
182
270
|
- test/petalbot_test.rb
|
183
271
|
- test/pinterest_test.rb
|
@@ -185,7 +273,8 @@ files:
|
|
185
273
|
homepage: https://github.com/alaz/legitbot
|
186
274
|
licenses:
|
187
275
|
- Apache-2.0
|
188
|
-
metadata:
|
276
|
+
metadata:
|
277
|
+
rubygems_mfa_required: 'true'
|
189
278
|
post_install_message:
|
190
279
|
rdoc_options:
|
191
280
|
- "--charset=UTF-8"
|
@@ -195,29 +284,29 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
195
284
|
requirements:
|
196
285
|
- - ">="
|
197
286
|
- !ruby/object:Gem::Version
|
198
|
-
version: 2.
|
287
|
+
version: 2.6.0
|
199
288
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
200
289
|
requirements:
|
201
290
|
- - ">="
|
202
291
|
- !ruby/object:Gem::Version
|
203
292
|
version: '0'
|
204
293
|
requirements: []
|
205
|
-
rubygems_version: 3.
|
294
|
+
rubygems_version: 3.2.32
|
206
295
|
signing_key:
|
207
296
|
specification_version: 4
|
208
297
|
summary: 'Validate requests from Web crawlers: impersonating or not?'
|
209
298
|
test_files:
|
210
|
-
- test/legitbot_test.rb
|
211
|
-
- test/legitbot/validators/domains_test.rb
|
212
|
-
- test/legitbot/validators/ip_ranges_test.rb
|
213
|
-
- test/pinterest_test.rb
|
214
|
-
- test/alexa_test.rb
|
215
299
|
- test/ahrefs_test.rb
|
216
|
-
- test/
|
217
|
-
- test/oracle_test.rb
|
218
|
-
- test/google_test.rb
|
300
|
+
- test/alexa_test.rb
|
219
301
|
- test/amazon_test.rb
|
220
|
-
- test/
|
302
|
+
- test/apple_test.rb
|
221
303
|
- test/botmatch_test.rb
|
222
304
|
- test/facebook_test.rb
|
305
|
+
- test/google_test.rb
|
306
|
+
- test/legitbot/validators/domains_test.rb
|
307
|
+
- test/legitbot/validators/ip_ranges_test.rb
|
308
|
+
- test/legitbot_test.rb
|
309
|
+
- test/oracle_test.rb
|
310
|
+
- test/petalbot_test.rb
|
311
|
+
- test/pinterest_test.rb
|
223
312
|
- test/twitter_test.rb
|