legitbot 0.3.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/build.yml +60 -0
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +2 -0
  5. data/.ruby-version +1 -0
  6. data/Gemfile +2 -0
  7. data/README.md +3 -1
  8. data/Rakefile +5 -3
  9. data/legitbot.gemspec +19 -18
  10. data/lib/legitbot.rb +4 -0
  11. data/lib/legitbot/ahrefs.rb +13 -8
  12. data/lib/legitbot/apple.rb +11 -11
  13. data/lib/legitbot/baidu.rb +5 -7
  14. data/lib/legitbot/bing.rb +5 -7
  15. data/lib/legitbot/botmatch.rb +17 -44
  16. data/lib/legitbot/config/resolver.rb +18 -0
  17. data/lib/legitbot/duckduckgo.rb +18 -7
  18. data/lib/legitbot/facebook.rb +8 -34
  19. data/lib/legitbot/google.rb +5 -8
  20. data/lib/legitbot/legitbot.rb +14 -9
  21. data/lib/legitbot/oracle.rb +10 -0
  22. data/lib/legitbot/pinterest.rb +5 -8
  23. data/lib/legitbot/twitter.rb +14 -0
  24. data/lib/legitbot/validators/domains.rb +71 -0
  25. data/lib/legitbot/validators/ip_ranges.rb +81 -0
  26. data/lib/legitbot/version.rb +3 -1
  27. data/lib/legitbot/yandex.rb +41 -12
  28. data/test/ahrefs_test.rb +16 -8
  29. data/test/apple_as_google_test.rb +9 -4
  30. data/test/apple_test.rb +11 -4
  31. data/test/botmatch_test.rb +4 -22
  32. data/test/facebook_test.rb +25 -10
  33. data/test/google_test.rb +24 -14
  34. data/test/legitbot/validators/domains_test.rb +58 -0
  35. data/test/legitbot/validators/ip_ranges_test.rb +113 -0
  36. data/test/legitbot_test.rb +8 -4
  37. data/test/oracle_test.rb +36 -0
  38. data/test/pinterest_test.rb +26 -14
  39. data/test/twitter_test.rb +36 -0
  40. metadata +87 -23
  41. data/.travis.yml +0 -12
metadata CHANGED
@@ -1,19 +1,42 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: legitbot
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexander Azarov
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-16 00:00:00.000000000 Z
11
+ date: 2020-06-18 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: augmented_interval_tree
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.1'
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 0.1.1
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '0.1'
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 0.1.1
13
33
  - !ruby/object:Gem::Dependency
14
34
  name: irrc
15
35
  requirement: !ruby/object:Gem::Requirement
16
36
  requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.2'
17
40
  - - ">="
18
41
  - !ruby/object:Gem::Version
19
42
  version: 0.2.1
@@ -21,74 +44,103 @@ dependencies:
21
44
  prerelease: false
22
45
  version_requirements: !ruby/object:Gem::Requirement
23
46
  requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '0.2'
24
50
  - - ">="
25
51
  - !ruby/object:Gem::Version
26
52
  version: 0.2.1
27
53
  - !ruby/object:Gem::Dependency
28
- name: augmented_interval_tree
54
+ name: bump
29
55
  requirement: !ruby/object:Gem::Requirement
30
56
  requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '0.8'
31
60
  - - ">="
32
61
  - !ruby/object:Gem::Version
33
- version: 0.1.1
34
- type: :runtime
62
+ version: 0.8.0
63
+ type: :development
35
64
  prerelease: false
36
65
  version_requirements: !ruby/object:Gem::Requirement
37
66
  requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '0.8'
38
70
  - - ">="
39
71
  - !ruby/object:Gem::Version
40
- version: 0.1.1
72
+ version: 0.8.0
41
73
  - !ruby/object:Gem::Dependency
42
- name: bump
74
+ name: minitest
43
75
  requirement: !ruby/object:Gem::Requirement
44
76
  requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '5.1'
45
80
  - - ">="
46
81
  - !ruby/object:Gem::Version
47
- version: '0'
82
+ version: 5.1.0
48
83
  type: :development
49
84
  prerelease: false
50
85
  version_requirements: !ruby/object:Gem::Requirement
51
86
  requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '5.1'
52
90
  - - ">="
53
91
  - !ruby/object:Gem::Version
54
- version: '0'
92
+ version: 5.1.0
55
93
  - !ruby/object:Gem::Dependency
56
94
  name: rake
57
95
  requirement: !ruby/object:Gem::Requirement
58
96
  requirements:
97
+ - - "~>"
98
+ - !ruby/object:Gem::Version
99
+ version: '12.3'
59
100
  - - ">="
60
101
  - !ruby/object:Gem::Version
61
- version: '0'
102
+ version: 12.3.0
62
103
  type: :development
63
104
  prerelease: false
64
105
  version_requirements: !ruby/object:Gem::Requirement
65
106
  requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '12.3'
66
110
  - - ">="
67
111
  - !ruby/object:Gem::Version
68
- version: '0'
112
+ version: 12.3.0
69
113
  - !ruby/object:Gem::Dependency
70
- name: minitest
114
+ name: rubocop
71
115
  requirement: !ruby/object:Gem::Requirement
72
116
  requirements:
117
+ - - "~>"
118
+ - !ruby/object:Gem::Version
119
+ version: '0.74'
73
120
  - - ">="
74
121
  - !ruby/object:Gem::Version
75
- version: '0'
122
+ version: 0.74.0
76
123
  type: :development
77
124
  prerelease: false
78
125
  version_requirements: !ruby/object:Gem::Requirement
79
126
  requirements:
127
+ - - "~>"
128
+ - !ruby/object:Gem::Version
129
+ version: '0.74'
80
130
  - - ">="
81
131
  - !ruby/object:Gem::Version
82
- version: '0'
83
- description: A library to make sure a Web request has been made by a real search engine,
84
- not a malicious agent
132
+ version: 0.74.0
133
+ description: Does Web request come from a real search engine or from an impersonating
134
+ agent?
85
135
  email: self@alaz.me
86
136
  executables: []
87
137
  extensions: []
88
138
  extra_rdoc_files: []
89
139
  files:
140
+ - ".github/workflows/build.yml"
90
141
  - ".gitignore"
91
- - ".travis.yml"
142
+ - ".rubocop.yml"
143
+ - ".ruby-version"
92
144
  - Gemfile
93
145
  - LICENSE.txt
94
146
  - README.md
@@ -100,11 +152,16 @@ files:
100
152
  - lib/legitbot/baidu.rb
101
153
  - lib/legitbot/bing.rb
102
154
  - lib/legitbot/botmatch.rb
155
+ - lib/legitbot/config/resolver.rb
103
156
  - lib/legitbot/duckduckgo.rb
104
157
  - lib/legitbot/facebook.rb
105
158
  - lib/legitbot/google.rb
106
159
  - lib/legitbot/legitbot.rb
160
+ - lib/legitbot/oracle.rb
107
161
  - lib/legitbot/pinterest.rb
162
+ - lib/legitbot/twitter.rb
163
+ - lib/legitbot/validators/domains.rb
164
+ - lib/legitbot/validators/ip_ranges.rb
108
165
  - lib/legitbot/version.rb
109
166
  - lib/legitbot/yandex.rb
110
167
  - test/ahrefs_test.rb
@@ -113,13 +170,17 @@ files:
113
170
  - test/botmatch_test.rb
114
171
  - test/facebook_test.rb
115
172
  - test/google_test.rb
173
+ - test/legitbot/validators/domains_test.rb
174
+ - test/legitbot/validators/ip_ranges_test.rb
116
175
  - test/legitbot_test.rb
176
+ - test/oracle_test.rb
117
177
  - test/pinterest_test.rb
178
+ - test/twitter_test.rb
118
179
  homepage: https://github.com/alaz/legitbot
119
180
  licenses:
120
181
  - Apache-2.0
121
182
  metadata: {}
122
- post_install_message:
183
+ post_install_message:
123
184
  rdoc_options:
124
185
  - "--charset=UTF-8"
125
186
  require_paths:
@@ -128,24 +189,27 @@ required_ruby_version: !ruby/object:Gem::Requirement
128
189
  requirements:
129
190
  - - ">="
130
191
  - !ruby/object:Gem::Version
131
- version: 2.3.0
192
+ version: 2.4.0
132
193
  required_rubygems_version: !ruby/object:Gem::Requirement
133
194
  requirements:
134
195
  - - ">="
135
196
  - !ruby/object:Gem::Version
136
197
  version: '0'
137
198
  requirements: []
138
- rubyforge_project:
139
- rubygems_version: 2.7.6.2
140
- signing_key:
199
+ rubygems_version: 3.1.2
200
+ signing_key:
141
201
  specification_version: 4
142
202
  summary: 'Validate requests from Web crawlers: impersonating or not?'
143
203
  test_files:
144
204
  - test/legitbot_test.rb
205
+ - test/legitbot/validators/domains_test.rb
206
+ - test/legitbot/validators/ip_ranges_test.rb
145
207
  - test/pinterest_test.rb
146
208
  - test/ahrefs_test.rb
147
209
  - test/apple_test.rb
148
210
  - test/apple_as_google_test.rb
211
+ - test/oracle_test.rb
149
212
  - test/google_test.rb
150
213
  - test/botmatch_test.rb
151
214
  - test/facebook_test.rb
215
+ - test/twitter_test.rb
data/.travis.yml DELETED
@@ -1,12 +0,0 @@
1
- sudo: false
2
- language: ruby
3
- cache: bundler
4
- rvm:
5
- - 2.5
6
- - 2.3
7
- - jruby-head
8
- jdk:
9
- - openjdk8
10
- before_install:
11
- - gem update --system
12
- - gem install bundler