crawler_detect 0.1.12 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6e03716868421dc64e5acb715b35599776fbe3951bbeb7e462892ed3a6a51d6
-  data.tar.gz: 10b00a68949482af5cb6064f538e0829f2bcb836fe5605da48b48f0af7bbad71
+  metadata.gz: 6535ee2f876e0b1e05444bf0bb4a7a9082fd70ba4a454d7b1e2e9737b4b84d41
+  data.tar.gz: ff05c16b5cd08416dfded4a8ca3c83ee73d2599dc2641aa8d212a7d3d72f3bd0
 SHA512:
-  metadata.gz: cb66a7c310ca791038ca199c68626d522f99cb4ddff57382b07ecb91b824e44292362351ccb71622051ece1d8c7a900019627fed731e46f5474507b8b20f6aaa
-  data.tar.gz: 3f4c2484f4155d975915dd21e1c62d6f8dd19ddda4023cff9530701cf628ddf94c10f5e097c3cb85ffbf3d71686bd91d9b9ac71da59cf47933549fd847d2c350
+  metadata.gz: 384c92b53a77f3b8280259897060f7fc74e6d0ac214d6469cfdd87a84c290286ae9bacace2875c2c033d69a77eb863cb7eee5053552d0c619b85f1f15ba9a79d
+  data.tar.gz: 3904d9beef1ff2211881408d7ab2c60caf44c2103d07f8188a5c8d9454e58d274f1779ca4d0ad0ca53a548631062dd9f4f97c32c0ee9e811191b17c117b59db5

data/.github/workflows/build.yml

@@ -0,0 +1,38 @@
+name: build
+
+on: [push]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: install gems
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+      - run: bundle exec rubocop
+  test:
+    needs: lint
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [2.5, 2.6, 2.7]
+        test-group: [1, 2, 3, 4]
+    name: Ruby ${{ matrix.ruby }}, test-group ${{ matrix.test-group }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: install gems
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+      - name: test
+        run: |
+          bundle exec parallel_rspec spec/ -n 4 --only-group ${{ matrix.test-group }} --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log

data/.rubocop.yml

@@ -1,174 +1,19 @@
-AllCops:
-  TargetRubyVersion: 2.2
-  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
-  # to ignore them, so only the ones explicitly set in this file are enabled.
-  DisabledByDefault: true
-  Exclude:
-  - '**/templates/**/*'
-  - '**/vendor/**/*'
-  - '**/vendor/**/.*'
-  - '**/node_modules/**/*'
-  - 'actionpack/lib/action_dispatch/journey/parser.rb'
+inherit_gem:
+  armitage-rubocop:
+    - lib/rubocop.general.yml
+    - lib/rubocop.rake.yml
+    - lib/rubocop.rspec.yml
 
-# Prefer assert_not_x over refute_x
-CustomCops/RefuteNot:
+AllCops:
+  TargetRubyVersion: 2.7.1
   Include:
-  - '**/test/**/*'
-
-# Prefer &&/|| over and/or.
-Style/AndOr:
-  Enabled: true
-
-# Do not use braces for hash literals when they are the last argument of a
-# method call.
-Style/BracesAroundHashParameters:
-  Enabled: true
-  EnforcedStyle: context_dependent
-
-# Align `when` with `case`.
-Layout/CaseIndentation:
-  Enabled: true
-
-# Align comments with method definitions.
-Layout/CommentIndentation:
-  Enabled: true
-
-Layout/ElseAlignment:
-  Enabled: true
-
-# Align `end` with the matching keyword or starting expression except for
-# assignments, where it should be aligned with the LHS.
-Layout/EndAlignment:
-  Enabled: true
-  EnforcedStyleAlignWith: variable
-  AutoCorrect: true
-
-Layout/EmptyLineAfterMagicComment:
-  Enabled: true
-
-# In a regular class definition, no empty lines around the body.
-Layout/EmptyLinesAroundClassBody:
-  Enabled: true
-
-# In a regular method definition, no empty lines around the body.
-Layout/EmptyLinesAroundMethodBody:
-  Enabled: true
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - crawler_detect.gemspec
+    - bin/console
 
-# In a regular module definition, no empty lines around the body.
-Layout/EmptyLinesAroundModuleBody:
-  Enabled: true
-
-Layout/FirstParameterIndentation:
-  Enabled: true
-
-# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
-Style/HashSyntax:
-  Enabled: true
-
-# Method definitions after `private` or `protected` isolated calls need one
-# extra level of indentation.
-Layout/IndentationConsistency:
-  Enabled: true
-  EnforcedStyle: rails
-
-# Two spaces, no tabs (for indentation).
-Layout/IndentationWidth:
-  Enabled: true
-
-Layout/LeadingCommentSpace:
-  Enabled: true
-
-Layout/SpaceAfterColon:
-  Enabled: true
-
-Layout/SpaceAfterComma:
-  Enabled: true
-
-Layout/SpaceAroundEqualsInParameterDefault:
-  Enabled: true
-
-Layout/SpaceAroundKeyword:
-  Enabled: true
-
-Layout/SpaceAroundOperators:
-  Enabled: true
-
-Layout/SpaceBeforeComma:
-  Enabled: true
-
-Layout/SpaceBeforeFirstArg:
-  Enabled: true
-
-Style/DefWithParentheses:
-  Enabled: true
-
-# Defining a method with parameters needs parentheses.
-Style/MethodDefParentheses:
-  Enabled: true
-
-Style/FrozenStringLiteralComment:
-  Enabled: true
-  EnforcedStyle: always
-  Exclude:
-  - 'actionview/test/**/*.builder'
-  - 'actionview/test/**/*.ruby'
-  - 'actionpack/test/**/*.builder'
-  - 'actionpack/test/**/*.ruby'
-  - 'activestorage/db/migrate/**/*.rb'
-  - 'db/migrate/**/*.rb'
-  - 'db/*.rb'
-
-# Use `foo {}` not `foo{}`.
-Layout/SpaceBeforeBlockBraces:
-  Enabled: true
-
-# Use `foo { bar }` not `foo {bar}`.
-Layout/SpaceInsideBlockBraces:
-  Enabled: true
-
-# Use `{ a: 1 }` not `{a:1}`.
-Layout/SpaceInsideHashLiteralBraces:
-  Enabled: true
-
-Layout/SpaceInsideParens:
-  Enabled: true
-
-# Check quotes usage according to lint rule below.
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
-
-# Detect hard tabs, no hard tabs.
-Layout/Tab:
-  Enabled: true
-
-# Blank lines should not have any spaces.
-Layout/TrailingBlankLines:
-  Enabled: true
-
-# No trailing whitespace.
-Layout/TrailingWhitespace:
-  Enabled: true
-
-# Use quotes for string literals when they are enough.
-Style/UnneededPercentQ:
-  Enabled: true
-
-# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
-Lint/RequireParentheses:
-  Enabled: true
-
-Lint/StringConversionInInterpolation:
-  Enabled: true
-
-Style/RedundantReturn:
-  Enabled: true
-  AllowMultipleReturnValues: true
-
-Style/Semicolon:
-  Enabled: true
-  AllowAsExpressionSeparator: true
-
-# Prefer Foo.method over Foo::method
-Style/ColonMethodCall:
-  Enabled: true

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# CrawlerDetect major changes
+
+This changelog **does not contain** raw data updates 
+but only major changes.
+
+1.1.0
+---------
+- Moves to Ruby's Stdlib JSON implementation to reduce dependencies
+
+1.0.0
+---------
+- Use raw JSON files instead of copy them to rb [#8]
+- Add CrawlerDetect::Config to make it possible to have own raw files [#8]
+- Add bin/update_raw_files to update raw files from PHP lib [#8] 
+- Add Changelog
+
+0.1.11
+---------
+- Add thread safety [#19]
+
+0.1.6
+---------
+- Strip crawler name [#10]
+
+0.1.2
+---------
+- Add parallel tests [#2]
+
+0.1.1
+---------
+- Fix: rack request
+
+0.1.0
+---------
+- init
+

data/Gemfile.lock

@@ -1,16 +1,25 @@
 PATH
   remote: .
   specs:
-    crawler_detect (0.1.11)
+    crawler_detect (1.1.0)
+      qonfig (~> 0.24)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.4.2)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    armitage-rubocop (0.82.0)
+      rubocop (= 0.82.0)
+      rubocop-performance (= 1.5.2)
+      rubocop-rails (= 2.5.2)
+      rubocop-rake (= 0.5.1)
+      rubocop-rspec (= 1.38.1)
+    ast (2.4.0)
     awesome_print (1.8.0)
     byebug (11.1.2)
     coderay (1.1.2)
@@ -21,11 +30,14 @@ GEM
       ruby-progressbar (~> 1.4)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
     method_source (1.0.0)
-    minitest (5.14.0)
+    minitest (5.14.1)
     parallel (1.19.1)
     parallel_tests (2.32.0)
       parallel
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -40,10 +52,13 @@ GEM
     pry-remote (0.1.8)
       pry (~> 0.9)
       slop (~> 3.0)
-    rack (2.2.2)
+    qonfig (0.24.1)
+    rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
+    rainbow (3.0.0)
     rake (13.0.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -57,17 +72,38 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.2)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.5.2)
+      activesupport
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (1.38.1)
+      rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     slop (3.6.0)
     thread_safe (0.3.6)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 5.2.0)
+  activesupport (~> 6.0.3)
+  armitage-rubocop (= 0.82)
   bundler (>= 1.15)
   crawler_detect!
   fuubar (~> 2.0)
@@ -78,4 +114,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   2.1.3
+   2.1.4

data/README.md

@@ -1,6 +1,6 @@
 # CrawlerDetect  
   
-[![Build Status](https://travis-ci.org/loadkpi/crawler_detect.svg?branch=master)](https://travis-ci.org/loadkpi/crawler_detect) [![Gem Version](https://badge.fury.io/rb/crawler_detect.svg)](https://badge.fury.io/rb/crawler_detect)
+![Build](https://github.com/loadkpi/crawler_detect/workflows/build/badge.svg?branch=master) [![Gem Version](https://badge.fury.io/rb/crawler_detect.svg)](https://badge.fury.io/rb/crawler_detect)
 
 ## About
 **CrawlerDetect** is a Ruby version of PHP class @[CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect). 
@@ -15,44 +15,59 @@ Comparing with other popular bot-detection gems:
 | Number of checked HTTP-headers | 10 | 1 | 1 |
 | Number of updates of bot-list *(1st half of 2018)* | 14 | 1 | 7 |
 
+In order to remain up-to-date, this gem does not accept any crawler data updates – any PRs to edit the crawler data should be offered to the original  [JayBizzle/CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect) project.
+
 ## Installation
 Add this line to your application's Gemfile:
 
 `gem 'crawler_detect'`
 ## Basic Usage
-```
+```ruby
 CrawlerDetect.is_crawler?("Bot user agent")
 => true
 ```
 Or if you need crawler name:
-```
+```ruby
 detector = CrawlerDetect.new("Googlebot/2.1 (http://www.google.com/bot.html)")
 detector.is_crawler?
-=> true
+# => true
 detector.crawler_name
-=> "Googlebot"
+# => "Googlebot"
 ```
 ## Rack::Request extension
 **Optionally** you can add additional methods for `request`:
-```
+```ruby
 request.is_crawler?
-=> false
+# => false
 request.crawler_name
-=> nil
+# => nil
 ```
 It's more flexible to use `request.is_crawler?` rather than `CrawlerDetect.is_crawler?` because it automatically checks 10 HTTP-headers, not only `HTTP_USER_AGENT`.
 
 Only one thing you have to do is to configure `Rack::CrawlerDetect` midleware:
 ###  Rails
-```
+```ruby
 class Application < Rails::Application
-  ...
+  # ...
   config.middleware.use Rack::CrawlerDetect
 end
 ```
-### Rake
-```
+### Rack
+```ruby
 use Rack::CrawlerDetect
 ```
+## Configuration
+In some cases you may want to use your own white-list, or black-list or list of http-headers to detect User-agent.
+
+It is possible to do via `CrawlerDetect::Config`. For example, you may have initializer like this:
+```ruby
+CrawlerDetect.setup! do |config|
+  config.raw_headers_path    = File.expand_path("crawlers/MyHeaders.json", __dir__)
+  config.raw_crawlers_path   = File.expand_path("crawlers/MyCrawlers.json", __dir__)
+  config.raw_exclusions_path = File.expand_path("crawlers/MyExclusions.json", __dir__)
+end
+```
+Make sure that your files are correct JSON files. 
+Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information. 
 ## License
 MIT License