crawler_detect 0.1.10 → 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 342d96d8f386511a05e7ab0894faefca3ea74a60910f378c2b5c1ab26bd530a8
4
- data.tar.gz: 5fb6e1759b67aeb2e2d644d6c48141ad02c3c74663a31468ae2f4c53b69df0fe
3
+ metadata.gz: c0dcfbc40c645fb77a672c9c455c5e306c75ea3d7086f6d9d1f67fc15172ca0d
4
+ data.tar.gz: fb32a563f4ec5790ff3d2ae68b6f6a8883a48ad383e6caec529b09355857b1f1
5
5
  SHA512:
6
- metadata.gz: 9ed795dd180037e8a131bc05b3a176488b70601373daabbd459a0f3c21bc49a240eaac4518148c50dc14bd5edd46f92f62e70453c2cbfd0d85002e3cdfc9dbdc
7
- data.tar.gz: 1e30cad0d3588db6a89fb95b04e5e09166a9742acc33f14c2dce2d56c25b5578f12e3c572dc2c1401fccf5d526518b851ce41f79e0a3b00f9ef6b361718ad14f
6
+ metadata.gz: 226d14061717db016ff02f28c5aff050b187a051caf2492d28297e01ee3409d6479683c8e31ed0a8f7235f8f5aadfea938e5f9266e3ead1ec2bafc3a2f7be54b
7
+ data.tar.gz: db86877f9680fe96f2cf9847f4ecab75fb65117d0d52992e901b25c2a6ac18c1a088e37429fe08b246b7e51df0ad65dd6b4fd0a2bf9e3f77d8d96133d63f359a
@@ -1,174 +1,19 @@
1
- AllCops:
2
- TargetRubyVersion: 2.2
3
- # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
4
- # to ignore them, so only the ones explicitly set in this file are enabled.
5
- DisabledByDefault: true
6
- Exclude:
7
- - '**/templates/**/*'
8
- - '**/vendor/**/*'
9
- - '**/vendor/**/.*'
10
- - '**/node_modules/**/*'
11
- - 'actionpack/lib/action_dispatch/journey/parser.rb'
1
+ inherit_gem:
2
+ armitage-rubocop:
3
+ - lib/rubocop.general.yml
4
+ - lib/rubocop.rake.yml
5
+ - lib/rubocop.rspec.yml
12
6
 
13
- # Prefer assert_not_x over refute_x
14
- CustomCops/RefuteNot:
7
+ AllCops:
8
+ TargetRubyVersion: 2.7.1
15
9
  Include:
16
- - '**/test/**/*'
17
-
18
- # Prefer &&/|| over and/or.
19
- Style/AndOr:
20
- Enabled: true
21
-
22
- # Do not use braces for hash literals when they are the last argument of a
23
- # method call.
24
- Style/BracesAroundHashParameters:
25
- Enabled: true
26
- EnforcedStyle: context_dependent
27
-
28
- # Align `when` with `case`.
29
- Layout/CaseIndentation:
30
- Enabled: true
31
-
32
- # Align comments with method definitions.
33
- Layout/CommentIndentation:
34
- Enabled: true
35
-
36
- Layout/ElseAlignment:
37
- Enabled: true
38
-
39
- # Align `end` with the matching keyword or starting expression except for
40
- # assignments, where it should be aligned with the LHS.
41
- Layout/EndAlignment:
42
- Enabled: true
43
- EnforcedStyleAlignWith: variable
44
- AutoCorrect: true
45
-
46
- Layout/EmptyLineAfterMagicComment:
47
- Enabled: true
48
-
49
- # In a regular class definition, no empty lines around the body.
50
- Layout/EmptyLinesAroundClassBody:
51
- Enabled: true
52
-
53
- # In a regular method definition, no empty lines around the body.
54
- Layout/EmptyLinesAroundMethodBody:
55
- Enabled: true
10
+ - lib/**/*.rb
11
+ - spec/**/*.rb
12
+ - Gemfile
13
+ - Rakefile
14
+ - crawler_detect.gemspec
15
+ - bin/console
56
16
 
57
- # In a regular module definition, no empty lines around the body.
58
- Layout/EmptyLinesAroundModuleBody:
59
- Enabled: true
60
-
61
- Layout/FirstParameterIndentation:
62
- Enabled: true
63
-
64
- # Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
65
- Style/HashSyntax:
66
- Enabled: true
67
-
68
- # Method definitions after `private` or `protected` isolated calls need one
69
- # extra level of indentation.
70
- Layout/IndentationConsistency:
71
- Enabled: true
72
- EnforcedStyle: rails
73
-
74
- # Two spaces, no tabs (for indentation).
75
- Layout/IndentationWidth:
76
- Enabled: true
77
-
78
- Layout/LeadingCommentSpace:
79
- Enabled: true
80
-
81
- Layout/SpaceAfterColon:
82
- Enabled: true
83
-
84
- Layout/SpaceAfterComma:
85
- Enabled: true
86
-
87
- Layout/SpaceAroundEqualsInParameterDefault:
88
- Enabled: true
89
-
90
- Layout/SpaceAroundKeyword:
91
- Enabled: true
92
-
93
- Layout/SpaceAroundOperators:
94
- Enabled: true
95
-
96
- Layout/SpaceBeforeComma:
97
- Enabled: true
98
-
99
- Layout/SpaceBeforeFirstArg:
100
- Enabled: true
101
-
102
- Style/DefWithParentheses:
103
- Enabled: true
104
-
105
- # Defining a method with parameters needs parentheses.
106
- Style/MethodDefParentheses:
107
- Enabled: true
108
-
109
- Style/FrozenStringLiteralComment:
110
- Enabled: true
111
- EnforcedStyle: always
112
- Exclude:
113
- - 'actionview/test/**/*.builder'
114
- - 'actionview/test/**/*.ruby'
115
- - 'actionpack/test/**/*.builder'
116
- - 'actionpack/test/**/*.ruby'
117
- - 'activestorage/db/migrate/**/*.rb'
118
- - 'db/migrate/**/*.rb'
119
- - 'db/*.rb'
120
-
121
- # Use `foo {}` not `foo{}`.
122
- Layout/SpaceBeforeBlockBraces:
123
- Enabled: true
124
-
125
- # Use `foo { bar }` not `foo {bar}`.
126
- Layout/SpaceInsideBlockBraces:
127
- Enabled: true
128
-
129
- # Use `{ a: 1 }` not `{a:1}`.
130
- Layout/SpaceInsideHashLiteralBraces:
131
- Enabled: true
132
-
133
- Layout/SpaceInsideParens:
134
- Enabled: true
135
-
136
- # Check quotes usage according to lint rule below.
137
17
  Style/StringLiterals:
138
18
  Enabled: true
139
19
  EnforcedStyle: double_quotes
140
-
141
- # Detect hard tabs, no hard tabs.
142
- Layout/Tab:
143
- Enabled: true
144
-
145
- # Blank lines should not have any spaces.
146
- Layout/TrailingBlankLines:
147
- Enabled: true
148
-
149
- # No trailing whitespace.
150
- Layout/TrailingWhitespace:
151
- Enabled: true
152
-
153
- # Use quotes for string literals when they are enough.
154
- Style/UnneededPercentQ:
155
- Enabled: true
156
-
157
- # Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
158
- Lint/RequireParentheses:
159
- Enabled: true
160
-
161
- Lint/StringConversionInInterpolation:
162
- Enabled: true
163
-
164
- Style/RedundantReturn:
165
- Enabled: true
166
- AllowMultipleReturnValues: true
167
-
168
- Style/Semicolon:
169
- Enabled: true
170
- AllowAsExpressionSeparator: true
171
-
172
- # Prefer Foo.method over Foo::method
173
- Style/ColonMethodCall:
174
- Enabled: true
@@ -1,16 +1,26 @@
1
- ---
2
- sudo: false
1
+ os: linux
2
+ dist: xenial
3
3
  language: ruby
4
4
  cache: bundler
5
+ before_install:
6
+ - gem install bundler
5
7
  env:
6
8
  - "TEST_GROUP=1"
7
9
  - "TEST_GROUP=2"
8
10
  - "TEST_GROUP=3"
9
11
  - "TEST_GROUP=4"
10
12
  rvm:
11
- - 2.3
12
13
  - 2.5
13
14
  - 2.6
14
- before_install: gem install bundler
15
- script:
16
- - bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
15
+ - 2.7
16
+ stages:
17
+ - lint
18
+ - test
19
+ script: bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
20
+ jobs:
21
+ fast_finish: true
22
+ include:
23
+ - stage: lint
24
+ rvm: 2.7
25
+ env: "TEST_GROUP=none"
26
+ script: bundle exec rubocop
@@ -0,0 +1,32 @@
1
+ # CrawlerDetect major changes
2
+
3
+ This changelog **does not contain** raw data updates
4
+ but only major changes.
5
+
6
+ 1.0.0
7
+ ---------
8
+ - Use raw JSON files instead of copy them to rb [#8]
9
+ - Add CrawlerDetect::Config to make it possible to have own raw files [#8]
10
+ - Add bin/update_raw_files to update raw files from PHP lib [#8]
11
+ - Add Changelog
12
+
13
+ 0.1.11
14
+ ---------
15
+ - Add thread safety [#19]
16
+
17
+ 0.1.6
18
+ ---------
19
+ - Strip crawler name [#10]
20
+
21
+ 0.1.2
22
+ ---------
23
+ - Add parallel tests [#2]
24
+
25
+ 0.1.1
26
+ ---------
27
+ - Fix: rack request
28
+
29
+ 0.1.0
30
+ ---------
31
+ - init
32
+
@@ -0,0 +1,119 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ crawler_detect (1.0.1)
5
+ oj (>= 3.0)
6
+ qonfig (~> 0.24)
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ activesupport (6.0.3.1)
12
+ concurrent-ruby (~> 1.0, >= 1.0.2)
13
+ i18n (>= 0.7, < 2)
14
+ minitest (~> 5.1)
15
+ tzinfo (~> 1.1)
16
+ zeitwerk (~> 2.2, >= 2.2.2)
17
+ armitage-rubocop (0.82.0)
18
+ rubocop (= 0.82.0)
19
+ rubocop-performance (= 1.5.2)
20
+ rubocop-rails (= 2.5.2)
21
+ rubocop-rake (= 0.5.1)
22
+ rubocop-rspec (= 1.38.1)
23
+ ast (2.4.0)
24
+ awesome_print (1.8.0)
25
+ byebug (11.1.2)
26
+ coderay (1.1.2)
27
+ concurrent-ruby (1.1.6)
28
+ diff-lcs (1.3)
29
+ fuubar (2.5.0)
30
+ rspec-core (~> 3.0)
31
+ ruby-progressbar (~> 1.4)
32
+ i18n (1.8.2)
33
+ concurrent-ruby (~> 1.0)
34
+ jaro_winkler (1.5.4)
35
+ method_source (1.0.0)
36
+ minitest (5.14.1)
37
+ oj (3.10.6)
38
+ parallel (1.19.1)
39
+ parallel_tests (2.32.0)
40
+ parallel
41
+ parser (2.7.1.2)
42
+ ast (~> 2.4.0)
43
+ pry (0.13.1)
44
+ coderay (~> 1.1)
45
+ method_source (~> 1.0)
46
+ pry-byebug (3.9.0)
47
+ byebug (~> 11.0)
48
+ pry (~> 0.13.0)
49
+ pry-meta (0.0.10)
50
+ awesome_print
51
+ pry
52
+ pry-byebug
53
+ pry-remote
54
+ pry-remote (0.1.8)
55
+ pry (~> 0.9)
56
+ slop (~> 3.0)
57
+ qonfig (0.24.1)
58
+ rack (2.2.3)
59
+ rack-test (1.1.0)
60
+ rack (>= 1.0, < 3)
61
+ rainbow (3.0.0)
62
+ rake (13.0.1)
63
+ rexml (3.2.4)
64
+ rspec (3.9.0)
65
+ rspec-core (~> 3.9.0)
66
+ rspec-expectations (~> 3.9.0)
67
+ rspec-mocks (~> 3.9.0)
68
+ rspec-core (3.9.1)
69
+ rspec-support (~> 3.9.1)
70
+ rspec-expectations (3.9.1)
71
+ diff-lcs (>= 1.2.0, < 2.0)
72
+ rspec-support (~> 3.9.0)
73
+ rspec-mocks (3.9.1)
74
+ diff-lcs (>= 1.2.0, < 2.0)
75
+ rspec-support (~> 3.9.0)
76
+ rspec-support (3.9.2)
77
+ rubocop (0.82.0)
78
+ jaro_winkler (~> 1.5.1)
79
+ parallel (~> 1.10)
80
+ parser (>= 2.7.0.1)
81
+ rainbow (>= 2.2.2, < 4.0)
82
+ rexml
83
+ ruby-progressbar (~> 1.7)
84
+ unicode-display_width (>= 1.4.0, < 2.0)
85
+ rubocop-performance (1.5.2)
86
+ rubocop (>= 0.71.0)
87
+ rubocop-rails (2.5.2)
88
+ activesupport
89
+ rack (>= 1.1)
90
+ rubocop (>= 0.72.0)
91
+ rubocop-rake (0.5.1)
92
+ rubocop
93
+ rubocop-rspec (1.38.1)
94
+ rubocop (>= 0.68.1)
95
+ ruby-progressbar (1.10.1)
96
+ slop (3.6.0)
97
+ thread_safe (0.3.6)
98
+ tzinfo (1.2.7)
99
+ thread_safe (~> 0.1)
100
+ unicode-display_width (1.7.0)
101
+ zeitwerk (2.3.0)
102
+
103
+ PLATFORMS
104
+ ruby
105
+
106
+ DEPENDENCIES
107
+ activesupport (~> 6.0.3)
108
+ armitage-rubocop (= 0.82)
109
+ bundler (>= 1.15)
110
+ crawler_detect!
111
+ fuubar (~> 2.0)
112
+ parallel_tests (~> 2.0)
113
+ pry-meta (~> 0.0.10)
114
+ rack-test (~> 1.1)
115
+ rake (>= 10.0)
116
+ rspec (~> 3.0)
117
+
118
+ BUNDLED WITH
119
+ 2.1.4
data/README.md CHANGED
@@ -54,5 +54,18 @@ end
54
54
  ```
55
55
  use Rack::CrawlerDetect
56
56
  ```
57
+ ## Configuration
58
+ In some cases you may want to use your own white-list, or black-list or list of http-headers to detect User-agent.
59
+
60
+ It is possible to do via `CrawlerDetect::Config`. For example, you may have initializer like this:
61
+ ```
62
+ CrawlerDetect.setup! do |config|
63
+ config.raw_headers_path = File.expand_path("crawlers/MyHeaders.json", __dir__)
64
+ config.raw_crawlers_path = File.expand_path("crawlers/MyCrawlers.json", __dir__)
65
+ config.raw_exclusions_path = File.expand_path("crawlers/MyExclusions.json", __dir__)
66
+ end
67
+ ```
68
+ Make sure that your files are correct JSON files.
69
+ Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information.
57
70
  ## License
58
71
  MIT License
@@ -0,0 +1,10 @@
1
+ #!/bin/bash
2
+
3
+ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4
+
5
+ wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
6
+ wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
7
+ wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
8
+
9
+ wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
10
+ wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path("../lib", __FILE__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require "crawler_detect/version"
6
6
 
@@ -17,14 +17,17 @@ Gem::Specification.new do |spec|
17
17
 
18
18
  # Specify which files should be added to the gem when it is released.
19
19
  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
20
- spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
20
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
21
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
22
22
  end
23
23
  spec.bindir = "exe"
24
24
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
25
25
  spec.require_paths = ["lib"]
26
26
 
27
- spec.add_development_dependency "activesupport", "~> 5.2.0"
27
+ spec.add_dependency "oj", ">= 3.0"
28
+ spec.add_dependency "qonfig", "~> 0.24"
29
+
30
+ spec.add_development_dependency "activesupport", "~> 6.0.3"
28
31
  spec.add_development_dependency "bundler", ">= 1.15"
29
32
  spec.add_development_dependency "fuubar", "~> 2.0"
30
33
  spec.add_development_dependency "parallel_tests", "~> 2.0"
@@ -32,4 +35,5 @@ Gem::Specification.new do |spec|
32
35
  spec.add_development_dependency "rack-test", "~> 1.1"
33
36
  spec.add_development_dependency "rake", ">= 10.0"
34
37
  spec.add_development_dependency "rspec", "~> 3.0"
38
+ spec.add_development_dependency "armitage-rubocop", "0.82"
35
39
  end