legitbot 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +1 -0
  3. data/README.md +1 -0
  4. data/legitbot.gemspec +19 -18
  5. data/lib/legitbot.rb +1 -0
  6. data/lib/legitbot/twitter.rb +14 -0
  7. data/lib/legitbot/version.rb +1 -1
  8. data/test/facebook_test.rb +1 -1
  9. data/test/twitter_test.rb +36 -0
  10. metadata +56 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dfc1b3322ff4f85957dabf6790d535f27f99feed47bdb4ff1bba65f6242d31a2
4
- data.tar.gz: 32e842cc3d297b3afda0ef9b265121a11c363857a726819f886b441e6c53a53c
3
+ metadata.gz: 86eb19b33835daaf208e4603d289ceb47c7ab68700a3bd08de99f60792d49798
4
+ data.tar.gz: 620efdf47110dfd96c07e1ffe148666b65d8121c639330f20b0b27f26b1a91d0
5
5
  SHA512:
6
- metadata.gz: 3654c256da13b37045425457a96ac9a8b41c5ae5c0cce49b7898170e2d23a66a5cb7e612503dc1d88dc2e1240dcb07c9ccc5d5aa8439f280144abe969dc0ae7b
7
- data.tar.gz: 554e120d1001a71f455aedcd4d30397b22130279a6f99f8e022d40ade50427590dd52e982820399852ec52fbb2c96b7ea1461f82d8e88cb89acc053097136b32
6
+ metadata.gz: 53fe5ac9f2fbcdbd229fe8dc32b2dade2671ecdb1b03d61a17c0d49139c0137790dbb97735987d4a9957cb0ae0563e6893ed8292c98dadec309ebc88f2013bfe
7
+ data.tar.gz: e1adf66298503e14032b13f8f16fe86fe5ed3a8fe1a3021bb165de7b2037a7051ba49ddd4ddc5aa396a34a627ef42e3c2239de1cb220eb8f07171718a0207a99
data/.rubocop.yml CHANGED
@@ -1,5 +1,6 @@
1
1
  AllCops:
2
2
  Include:
3
+ - '**/*.gemspec'
3
4
  - '**/Gemfile'
4
5
  - '**/Rakefile'
5
6
  - 'lib/**/*.rb'
data/README.md CHANGED
@@ -51,6 +51,7 @@ end
51
51
  * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
52
52
  * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
53
53
  * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
54
+ * [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
54
55
  * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
55
56
 
56
57
  ## License
data/legitbot.gemspec CHANGED
@@ -1,28 +1,29 @@
1
- # encoding: utf-8
2
- $LOAD_PATH.push File.expand_path("../lib", __FILE__)
3
- require "legitbot/version"
1
+ # frozen_string_literal: true
2
+
3
+ $LOAD_PATH.push File.expand_path('lib', __dir__)
4
+ require 'legitbot/version'
4
5
 
5
6
  Gem::Specification.new do |spec|
6
7
  spec.name = 'legitbot'
7
8
  spec.version = Legitbot::VERSION
8
9
  spec.license = 'Apache-2.0'
9
10
 
10
- spec.author = "Alexander Azarov"
11
- spec.email = "self@alaz.me"
12
- spec.homepage = "https://github.com/alaz/legitbot"
13
- spec.summary = %q{Validate requests from Web crawlers: impersonating or not?}
14
- spec.description = "A library to make sure a Web request has been "\
15
- "made by a real search engine, not a malicious agent"
11
+ spec.author = 'Alexander Azarov'
12
+ spec.email = 'self@alaz.me'
13
+ spec.homepage = 'https://github.com/alaz/legitbot'
14
+ spec.summary = 'Validate requests from Web crawlers: impersonating or not?'
15
+ spec.description = 'A library to make sure a Web request has been '\
16
+ 'made by a real search engine, not a malicious agent'
16
17
 
17
18
  spec.required_ruby_version = '>= 2.3.0'
18
- spec.add_dependency "irrc", ">= 0.2.1"
19
- spec.add_dependency "augmented_interval_tree", ">= 0.1.1"
20
- spec.add_development_dependency "bump", '>= 0.8.0'
21
- spec.add_development_dependency "rake", '>= 12.3.0'
22
- spec.add_development_dependency "rubocop", '>= 0.74.0'
23
- spec.add_development_dependency "minitest", '>= 5.1.0'
19
+ spec.add_dependency 'augmented_interval_tree', '~> 0.1', '>= 0.1.1'
20
+ spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
21
+ spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
22
+ spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
23
+ spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.0'
24
+ spec.add_development_dependency 'rubocop', '~> 0.74', '>= 0.74.0'
24
25
 
25
- spec.files = `git ls-files`.split($/)
26
- spec.rdoc_options = ["--charset=UTF-8"]
27
- spec.test_files = Dir.glob("test/**/*")
26
+ spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
27
+ spec.rdoc_options = ['--charset=UTF-8']
28
+ spec.test_files = Dir.glob('test/**/*')
28
29
  end
data/lib/legitbot.rb CHANGED
@@ -11,4 +11,5 @@ require_relative 'legitbot/duckduckgo'
11
11
  require_relative 'legitbot/facebook'
12
12
  require_relative 'legitbot/google'
13
13
  require_relative 'legitbot/pinterest'
14
+ require_relative 'legitbot/twitter'
14
15
  require_relative 'legitbot/yandex'
data/lib/legitbot/twitter.rb ADDED
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Legitbot # :nodoc:
4
+ # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started
5
+ # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards
6
+ class Twitter < BotMatch
7
+ ip_ranges %w[
8
+ 199.16.156.0/22
9
+ 199.59.148.0/22
10
+ ]
11
+ end
12
+
13
+ rule Legitbot::Twitter, %w[Twitterbot]
14
+ end
data/lib/legitbot/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Legitbot
4
- VERSION = '0.4.0'
4
+ VERSION = '0.4.1'
5
5
  end
data/test/facebook_test.rb CHANGED
@@ -55,7 +55,7 @@ class FacebookTest < Minitest::Test
55
55
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
56
56
  '92.243.181.7'
57
57
  ) do |bot|
58
- assert_equal :facebook, bot.detected_as
58
+ assert %i[facebook twitter].include?(bot.detected_as)
59
59
  assert bot.fake?, msg: 'fake Facebook'
60
60
  end
61
61
  end
data/test/twitter_test.rb ADDED
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'minitest/autorun'
4
+ require 'legitbot'
5
+
6
+ class TwitterTest < Minitest::Test
7
+ def test_malicious_ip
8
+ ip = '149.210.164.47'
9
+ match = Legitbot::Twitter.new ip
10
+ assert !match.valid?, msg: "#{ip} is not a real Twitter IP"
11
+ end
12
+
13
+ def test_valid_ip
14
+ ip = '199.16.156.125'
15
+ match = Legitbot::Twitter.new ip
16
+ assert match.valid?, msg: "#{ip} is a valid Twitter IP"
17
+ end
18
+
19
+ def test_malicious_ua
20
+ bot = Legitbot.bot(
21
+ 'Twitterbot/1.0',
22
+ '149.210.164.47'
23
+ )
24
+ assert bot, msg: 'Twitter detected from User-Agent'
25
+ assert !bot.valid?, msg: 'Not a valid Twitter'
26
+ end
27
+
28
+ def test_valid_ua
29
+ bot = Legitbot.bot(
30
+ 'Twitterbot/1.0',
31
+ '199.16.156.125'
32
+ )
33
+ assert bot, msg: 'Twitter detected from User-Agent'
34
+ assert bot.valid?, msg: 'Valid Twitter'
35
+ end
36
+ end
metadata CHANGED
@@ -1,47 +1,62 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: legitbot
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexander Azarov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-18 00:00:00.000000000 Z
11
+ date: 2019-10-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: irrc
14
+ name: augmented_interval_tree
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.1'
17
20
  - - ">="
18
21
  - !ruby/object:Gem::Version
19
- version: 0.2.1
22
+ version: 0.1.1
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '0.1'
24
30
  - - ">="
25
31
  - !ruby/object:Gem::Version
26
- version: 0.2.1
32
+ version: 0.1.1
27
33
  - !ruby/object:Gem::Dependency
28
- name: augmented_interval_tree
34
+ name: irrc
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.2'
31
40
  - - ">="
32
41
  - !ruby/object:Gem::Version
33
- version: 0.1.1
42
+ version: 0.2.1
34
43
  type: :runtime
35
44
  prerelease: false
36
45
  version_requirements: !ruby/object:Gem::Requirement
37
46
  requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '0.2'
38
50
  - - ">="
39
51
  - !ruby/object:Gem::Version
40
- version: 0.1.1
52
+ version: 0.2.1
41
53
  - !ruby/object:Gem::Dependency
42
54
  name: bump
43
55
  requirement: !ruby/object:Gem::Requirement
44
56
  requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '0.8'
45
60
  - - ">="
46
61
  - !ruby/object:Gem::Version
47
62
  version: 0.8.0
@@ -49,51 +64,72 @@ dependencies:
49
64
  prerelease: false
50
65
  version_requirements: !ruby/object:Gem::Requirement
51
66
  requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '0.8'
52
70
  - - ">="
53
71
  - !ruby/object:Gem::Version
54
72
  version: 0.8.0
55
73
  - !ruby/object:Gem::Dependency
56
- name: rake
74
+ name: minitest
57
75
  requirement: !ruby/object:Gem::Requirement
58
76
  requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '5.1'
59
80
  - - ">="
60
81
  - !ruby/object:Gem::Version
61
- version: 12.3.0
82
+ version: 5.1.0
62
83
  type: :development
63
84
  prerelease: false
64
85
  version_requirements: !ruby/object:Gem::Requirement
65
86
  requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '5.1'
66
90
  - - ">="
67
91
  - !ruby/object:Gem::Version
68
- version: 12.3.0
92
+ version: 5.1.0
69
93
  - !ruby/object:Gem::Dependency
70
- name: rubocop
94
+ name: rake
71
95
  requirement: !ruby/object:Gem::Requirement
72
96
  requirements:
97
+ - - "~>"
98
+ - !ruby/object:Gem::Version
99
+ version: '12.3'
73
100
  - - ">="
74
101
  - !ruby/object:Gem::Version
75
- version: 0.74.0
102
+ version: 12.3.0
76
103
  type: :development
77
104
  prerelease: false
78
105
  version_requirements: !ruby/object:Gem::Requirement
79
106
  requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '12.3'
80
110
  - - ">="
81
111
  - !ruby/object:Gem::Version
82
- version: 0.74.0
112
+ version: 12.3.0
83
113
  - !ruby/object:Gem::Dependency
84
- name: minitest
114
+ name: rubocop
85
115
  requirement: !ruby/object:Gem::Requirement
86
116
  requirements:
117
+ - - "~>"
118
+ - !ruby/object:Gem::Version
119
+ version: '0.74'
87
120
  - - ">="
88
121
  - !ruby/object:Gem::Version
89
- version: 5.1.0
122
+ version: 0.74.0
90
123
  type: :development
91
124
  prerelease: false
92
125
  version_requirements: !ruby/object:Gem::Requirement
93
126
  requirements:
127
+ - - "~>"
128
+ - !ruby/object:Gem::Version
129
+ version: '0.74'
94
130
  - - ">="
95
131
  - !ruby/object:Gem::Version
96
- version: 5.1.0
132
+ version: 0.74.0
97
133
  description: A library to make sure a Web request has been made by a real search engine,
98
134
  not a malicious agent
99
135
  email: self@alaz.me
@@ -121,6 +157,7 @@ files:
121
157
  - lib/legitbot/google.rb
122
158
  - lib/legitbot/legitbot.rb
123
159
  - lib/legitbot/pinterest.rb
160
+ - lib/legitbot/twitter.rb
124
161
  - lib/legitbot/validators/domains.rb
125
162
  - lib/legitbot/validators/ip_ranges.rb
126
163
  - lib/legitbot/version.rb
@@ -135,6 +172,7 @@ files:
135
172
  - test/legitbot/validators/ip_ranges_test.rb
136
173
  - test/legitbot_test.rb
137
174
  - test/pinterest_test.rb
175
+ - test/twitter_test.rb
138
176
  homepage: https://github.com/alaz/legitbot
139
177
  licenses:
140
178
  - Apache-2.0
@@ -171,3 +209,4 @@ test_files:
171
209
  - test/google_test.rb
172
210
  - test/botmatch_test.rb
173
211
  - test/facebook_test.rb
212
+ - test/twitter_test.rb