legitbot 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/README.md +17 -9
- data/lib/legitbot/botmatch.rb +8 -2
- data/lib/legitbot/legitbot.rb +5 -5
- data/lib/legitbot/pinterest.rb +1 -1
- data/lib/legitbot/version.rb +1 -1
- data/test/botmatch_test.rb +5 -0
- data/test/legitbot_test.rb +1 -0
- data/test/pinterest_test.rb +5 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d526ed7ca64658503c66d7c291c8a24f5a9f8fdecb47899eca06d4135ccf4165
|
4
|
+
data.tar.gz: 9ba9bc7cebe5137e5a018366ccc63f869864f264b8a4988e4f971b0399d6bf39
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dc2814147aa8a02dc14ca39831b62ce5dab4d6506e4c7868d9b1938747858c09dd303e3ff955183dff853fdbb4d2f5e7469e6b433af71191ffb519c9cce403a9
|
7
|
+
data.tar.gz: 9b523df0e7d980cc4002197e63d9b76c95b7ed06504a431b2a7dcc3a61a0d126a786996e7e4031727aec92fbba726ff7e507d4d99e8e121780ce822c12706d99
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# Legitbot [![Build Status](https://secure.travis-ci.org/alaz/legitbot.png?branch=master)](http://travis-ci.org/alaz/legitbot) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
|
2
2
|
|
3
|
-
Ruby gem to check
|
4
|
-
engine. This can of
|
5
|
-
|
3
|
+
Ruby gem to check that an IP belongs to a bot, typically a search
|
4
|
+
engine. This can be of help in protecting a web site from fake search
|
5
|
+
engines.
|
6
6
|
|
7
7
|
## Usage
|
8
8
|
|
@@ -14,7 +14,7 @@ bot = Legitbot.bot(userAgent, ip)
|
|
14
14
|
```
|
15
15
|
|
16
16
|
`bot` will be `nil` if no bot signature was found in the `User-Agent`. Otherwise,
|
17
|
-
it will be an
|
17
|
+
it will be an object with methods
|
18
18
|
|
19
19
|
```ruby
|
20
20
|
bot.detected_as # => :google
|
@@ -22,7 +22,7 @@ bot.valid? # => true
|
|
22
22
|
bot.fake? # => false
|
23
23
|
```
|
24
24
|
|
25
|
-
Sometimes you already know what search engine to expect. For example, you
|
25
|
+
Sometimes you already know what search engine to expect. For example, you might
|
26
26
|
be using [rack-attack](https://github.com/kickstarter/rack-attack):
|
27
27
|
|
28
28
|
```ruby
|
@@ -31,6 +31,16 @@ Rack::Attack.blocklist("fake Googlebot") do |req|
|
|
31
31
|
end
|
32
32
|
```
|
33
33
|
|
34
|
+
Or if you do not like all these nasty crawlers stealing your content or
|
35
|
+
maybe evaluating it and getting ready to invade your site with spammers,
|
36
|
+
then block them all:
|
37
|
+
|
38
|
+
```ruby
|
39
|
+
Rack::Attack.blocklist 'fake search engines' do |request|
|
40
|
+
Legitbot.bot(request.user_agent, request.ip)&.fake?
|
41
|
+
end
|
42
|
+
```
|
43
|
+
|
34
44
|
## Supported
|
35
45
|
|
36
46
|
* [Applebot](https://support.apple.com/en-us/HT204683)
|
@@ -48,9 +58,7 @@ Apache 2.0
|
|
48
58
|
|
49
59
|
## References
|
50
60
|
|
51
|
-
*
|
61
|
+
* Play Framework variant in Scala: [play-legitbot](https://github.com/osinka/play-legitbot)
|
52
62
|
* Article [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
|
53
|
-
* [Voight-Kampff](https://github.com/biola/Voight-Kampff) is a Ruby gem
|
63
|
+
* [Voight-Kampff](https://github.com/biola/Voight-Kampff) is a Ruby gem that
|
54
64
|
detects bots by `User-Agent`
|
55
|
-
* [browser](https://github.com/fnando/browser) is a Ruby gem which may tell
|
56
|
-
you if the request comes from a search engine.
|
data/lib/legitbot/botmatch.rb
CHANGED
@@ -17,26 +17,32 @@ module Legitbot
|
|
17
17
|
# the reverse name
|
18
18
|
def reverse_domain
|
19
19
|
@reverse_domain ||= @dns.getname(@ip)
|
20
|
+
rescue Resolv::ResolvError
|
21
|
+
@reverse_domain ||= nil
|
20
22
|
end
|
21
23
|
|
22
24
|
##
|
23
25
|
# Returns a String with the reverse name
|
24
26
|
def reverse_name
|
25
|
-
reverse_domain
|
27
|
+
reverse_domain&.to_s
|
26
28
|
end
|
27
29
|
|
28
30
|
##
|
29
31
|
# Returns a String with IP created from the reverse name
|
30
32
|
def reversed_ip
|
33
|
+
return nil if reverse_name.nil?
|
34
|
+
|
31
35
|
@reverse_ip ||= @dns.getaddress(reverse_name)
|
32
36
|
@reverse_ip.to_s
|
33
37
|
end
|
34
38
|
|
35
39
|
def reverse_resolves?
|
36
|
-
|
40
|
+
@ip == reversed_ip
|
37
41
|
end
|
38
42
|
|
39
43
|
def subdomain_of?(*domains)
|
44
|
+
return false if reverse_name.nil?
|
45
|
+
|
40
46
|
domains.any? { |d|
|
41
47
|
reverse_domain.subdomain_of? Resolv::DNS::Name.create(d)
|
42
48
|
}
|
data/lib/legitbot/legitbot.rb
CHANGED
@@ -18,13 +18,13 @@ module Legitbot
|
|
18
18
|
rule[:class].new(ip, resolver_config)
|
19
19
|
}
|
20
20
|
|
21
|
-
|
22
|
-
|
21
|
+
selected = bots.select { |b| b.valid? }.first if bots.size > 1
|
22
|
+
selected = bots.first if selected.nil?
|
23
23
|
|
24
|
-
if
|
25
|
-
yield
|
24
|
+
if selected && block_given?
|
25
|
+
yield selected
|
26
26
|
else
|
27
|
-
|
27
|
+
selected
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
data/lib/legitbot/pinterest.rb
CHANGED
data/lib/legitbot/version.rb
CHANGED
data/test/botmatch_test.rb
CHANGED
@@ -17,6 +17,11 @@ class BotMatchTest < Minitest::Test
|
|
17
17
|
assert_equal true, match.reverse_resolves?
|
18
18
|
end
|
19
19
|
|
20
|
+
def test_reverse_doesnt_resolve
|
21
|
+
match = Legitbot::BotMatch.new "5.140.70.64"
|
22
|
+
assert !match.reverse_resolves?
|
23
|
+
end
|
24
|
+
|
20
25
|
def test_valid_class_syntax
|
21
26
|
assert Legitbot::Google.valid?("66.249.64.141"), msg: "Valid Googlebot"
|
22
27
|
assert Legitbot::Google.fake?("149.210.164.47"), msg: "Fake Googlebot"
|
data/test/legitbot_test.rb
CHANGED
@@ -4,6 +4,7 @@ require 'legitbot'
|
|
4
4
|
class LegitbotTest < Minitest::Test
|
5
5
|
def test_rules
|
6
6
|
assert !Legitbot.bot("Firefox", "127.0.0.1"), msg: "Not a bot"
|
7
|
+
assert Legitbot.bot("Googlebot", "5.140.70.64"), msg: "No reverse resolve, bot"
|
7
8
|
|
8
9
|
Legitbot.bot("Firefox", "127.0.0.1") do |bot|
|
9
10
|
flunk "No bot Firefox is possible"
|
data/test/pinterest_test.rb
CHANGED
@@ -30,6 +30,11 @@ class PinterestTest < Minitest::Test
|
|
30
30
|
assert bot.valid?, msg: "Valid Pinterest"
|
31
31
|
end
|
32
32
|
|
33
|
+
def test_android_not_bot
|
34
|
+
bot = Legitbot.bot("Mozilla/5.0 (Linux; Android 8.0.0; SM-G965F Build/R16NW; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.64 Mobile Safari/537.36 [Pinterest/Android]", "85.117.106.133")
|
35
|
+
assert_nil bot
|
36
|
+
end
|
37
|
+
|
33
38
|
def test_engine_name
|
34
39
|
bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "54.236.1.11")
|
35
40
|
assert_equal :pinterest, bot.detected_as
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: legitbot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Azarov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: irrc
|