crawler_detect 0.1.12 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6e03716868421dc64e5acb715b35599776fbe3951bbeb7e462892ed3a6a51d6
-  data.tar.gz: 10b00a68949482af5cb6064f538e0829f2bcb836fe5605da48b48f0af7bbad71
+  metadata.gz: 8bdca229af61e7f9a0c2cd3b417db317d0ef8a4b6635137e7c5b7c6d8244c5e7
+  data.tar.gz: 959bf5817c54bf37c68c2555acf1a880d0ed4fdfab5e6a1d5678a68badd80557
 SHA512:
-  metadata.gz: cb66a7c310ca791038ca199c68626d522f99cb4ddff57382b07ecb91b824e44292362351ccb71622051ece1d8c7a900019627fed731e46f5474507b8b20f6aaa
-  data.tar.gz: 3f4c2484f4155d975915dd21e1c62d6f8dd19ddda4023cff9530701cf628ddf94c10f5e097c3cb85ffbf3d71686bd91d9b9ac71da59cf47933549fd847d2c350
+  metadata.gz: f69c6e4306bd0e09dc632d1c8f7f626a7bb4cc858ff85c17eb7761466eac7a9161dc79976b66ac6817e7169c2358d662c77a9c98b51802eeb37e597b7c40a9d7
+  data.tar.gz: 4798889b93af2b6441f632bc53de96cbcfc4a1d5f4242eb07fb95265d4359cbf304a860eb263e361022b92416bf2cbfef83d50f10f41f69264be659dc02578ac

data/.rubocop.yml

@@ -1,174 +1,19 @@
-AllCops:
-  TargetRubyVersion: 2.2
-  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
-  # to ignore them, so only the ones explicitly set in this file are enabled.
-  DisabledByDefault: true
-  Exclude:
-  - '**/templates/**/*'
-  - '**/vendor/**/*'
-  - '**/vendor/**/.*'
-  - '**/node_modules/**/*'
-  - 'actionpack/lib/action_dispatch/journey/parser.rb'
+inherit_gem:
+  armitage-rubocop:
+    - lib/rubocop.general.yml
+    - lib/rubocop.rake.yml
+    - lib/rubocop.rspec.yml
 
-# Prefer assert_not_x over refute_x
-CustomCops/RefuteNot:
+AllCops:
+  TargetRubyVersion: 2.7.1
   Include:
-  - '**/test/**/*'
-
-# Prefer &&/|| over and/or.
-Style/AndOr:
-  Enabled: true
-
-# Do not use braces for hash literals when they are the last argument of a
-# method call.
-Style/BracesAroundHashParameters:
-  Enabled: true
-  EnforcedStyle: context_dependent
-
-# Align `when` with `case`.
-Layout/CaseIndentation:
-  Enabled: true
-
-# Align comments with method definitions.
-Layout/CommentIndentation:
-  Enabled: true
-
-Layout/ElseAlignment:
-  Enabled: true
-
-# Align `end` with the matching keyword or starting expression except for
-# assignments, where it should be aligned with the LHS.
-Layout/EndAlignment:
-  Enabled: true
-  EnforcedStyleAlignWith: variable
-  AutoCorrect: true
-
-Layout/EmptyLineAfterMagicComment:
-  Enabled: true
-
-# In a regular class definition, no empty lines around the body.
-Layout/EmptyLinesAroundClassBody:
-  Enabled: true
-
-# In a regular method definition, no empty lines around the body.
-Layout/EmptyLinesAroundMethodBody:
-  Enabled: true
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - crawler_detect.gemspec
+    - bin/console
 
-# In a regular module definition, no empty lines around the body.
-Layout/EmptyLinesAroundModuleBody:
-  Enabled: true
-
-Layout/FirstParameterIndentation:
-  Enabled: true
-
-# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
-Style/HashSyntax:
-  Enabled: true
-
-# Method definitions after `private` or `protected` isolated calls need one
-# extra level of indentation.
-Layout/IndentationConsistency:
-  Enabled: true
-  EnforcedStyle: rails
-
-# Two spaces, no tabs (for indentation).
-Layout/IndentationWidth:
-  Enabled: true
-
-Layout/LeadingCommentSpace:
-  Enabled: true
-
-Layout/SpaceAfterColon:
-  Enabled: true
-
-Layout/SpaceAfterComma:
-  Enabled: true
-
-Layout/SpaceAroundEqualsInParameterDefault:
-  Enabled: true
-
-Layout/SpaceAroundKeyword:
-  Enabled: true
-
-Layout/SpaceAroundOperators:
-  Enabled: true
-
-Layout/SpaceBeforeComma:
-  Enabled: true
-
-Layout/SpaceBeforeFirstArg:
-  Enabled: true
-
-Style/DefWithParentheses:
-  Enabled: true
-
-# Defining a method with parameters needs parentheses.
-Style/MethodDefParentheses:
-  Enabled: true
-
-Style/FrozenStringLiteralComment:
-  Enabled: true
-  EnforcedStyle: always
-  Exclude:
-  - 'actionview/test/**/*.builder'
-  - 'actionview/test/**/*.ruby'
-  - 'actionpack/test/**/*.builder'
-  - 'actionpack/test/**/*.ruby'
-  - 'activestorage/db/migrate/**/*.rb'
-  - 'db/migrate/**/*.rb'
-  - 'db/*.rb'
-
-# Use `foo {}` not `foo{}`.
-Layout/SpaceBeforeBlockBraces:
-  Enabled: true
-
-# Use `foo { bar }` not `foo {bar}`.
-Layout/SpaceInsideBlockBraces:
-  Enabled: true
-
-# Use `{ a: 1 }` not `{a:1}`.
-Layout/SpaceInsideHashLiteralBraces:
-  Enabled: true
-
-Layout/SpaceInsideParens:
-  Enabled: true
-
-# Check quotes usage according to lint rule below.
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
-
-# Detect hard tabs, no hard tabs.
-Layout/Tab:
-  Enabled: true
-
-# Blank lines should not have any spaces.
-Layout/TrailingBlankLines:
-  Enabled: true
-
-# No trailing whitespace.
-Layout/TrailingWhitespace:
-  Enabled: true
-
-# Use quotes for string literals when they are enough.
-Style/UnneededPercentQ:
-  Enabled: true
-
-# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
-Lint/RequireParentheses:
-  Enabled: true
-
-Lint/StringConversionInInterpolation:
-  Enabled: true
-
-Style/RedundantReturn:
-  Enabled: true
-  AllowMultipleReturnValues: true
-
-Style/Semicolon:
-  Enabled: true
-  AllowAsExpressionSeparator: true
-
-# Prefer Foo.method over Foo::method
-Style/ColonMethodCall:
-  Enabled: true

data/.travis.yml

@@ -1,7 +1,9 @@
----
-sudo: false
+os: linux
+dist: xenial
 language: ruby
 cache: bundler
+before_install:
+  - gem install bundler
 env:
   - "TEST_GROUP=1"
   - "TEST_GROUP=2"
@@ -11,6 +13,14 @@ rvm:
   - 2.5
   - 2.6
   - 2.7
-before_install: gem install bundler
-script:
-  - bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+stages:
+  - lint
+  - test
+script: bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+jobs:
+  fast_finish: true
+  include:
+    - stage: lint
+      rvm: 2.7
+      env: "TEST_GROUP=none"
+      script: bundle exec rubocop

data/CHANGELOG.md

@@ -0,0 +1,32 @@
+# CrawlerDetect major changes
+
+This changelog **does not contain** raw data updates 
+but only major changes.
+
+1.0.0
+---------
+- Use raw JSON files instead of copy them to rb [#8]
+- Add CrawlerDetect::Config to make it possible to have own raw files [#8]
+- Add bin/update_raw_files to update raw files from PHP lib [#8] 
+- Add Changelog
+
+0.1.11
+---------
+- Add thread safety [#19]
+
+0.1.6
+---------
+- Strip crawler name [#10]
+
+0.1.2
+---------
+- Add parallel tests [#2]
+
+0.1.1
+---------
+- Fix: rack request
+
+0.1.0
+---------
+- init
+

data/Gemfile.lock

@@ -1,7 +1,9 @@
 PATH
   remote: .
   specs:
-    crawler_detect (0.1.11)
+    crawler_detect (1.0.0)
+      oj (>= 3.0)
+      qonfig (~> 0.24)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +13,13 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    armitage-rubocop (0.82.0)
+      rubocop (= 0.82.0)
+      rubocop-performance (= 1.5.2)
+      rubocop-rails (= 2.5.2)
+      rubocop-rake (= 0.5.1)
+      rubocop-rspec (= 1.38.1)
+    ast (2.4.0)
     awesome_print (1.8.0)
     byebug (11.1.2)
     coderay (1.1.2)
@@ -21,11 +30,15 @@ GEM
       ruby-progressbar (~> 1.4)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
     method_source (1.0.0)
     minitest (5.14.0)
+    oj (3.10.6)
     parallel (1.19.1)
     parallel_tests (2.32.0)
       parallel
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -40,10 +53,13 @@ GEM
     pry-remote (0.1.8)
       pry (~> 0.9)
       slop (~> 3.0)
+    qonfig (0.24.1)
     rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
+    rainbow (3.0.0)
     rake (13.0.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -57,17 +73,37 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.2)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.5.2)
+      activesupport
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (1.38.1)
+      rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     slop (3.6.0)
     thread_safe (0.3.6)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   activesupport (~> 5.2.0)
+  armitage-rubocop (= 0.82)
   bundler (>= 1.15)
   crawler_detect!
   fuubar (~> 2.0)
@@ -78,4 +114,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   2.1.3
+   2.1.4

data/README.md

@@ -54,5 +54,18 @@ end
 ```
 use Rack::CrawlerDetect
 ```
+## Configuration
+In some cases you may want to use your own white-list, or black-list or list of http-headers to detect User-agent.
+
+It is possible to do via `CrawlerDetect::Config`. For example, you may have initializer like this:
+```
+CrawlerDetect.setup! do |config|
+  config.raw_headers_path    = File.expand_path("crawlers/MyHeaders.json", __dir__)
+  config.raw_crawlers_path   = File.expand_path("crawlers/MyCrawlers.json", __dir__)
+  config.raw_exclusions_path = File.expand_path("crawlers/MyExclusions.json", __dir__)
+end
+```
+Make sure that your files are correct JSON files. 
+Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information. 
 ## License
 MIT License

data/bin/update_raw_files

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
+
+wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
+wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt

data/crawler_detect.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "crawler_detect/version"
 
@@ -17,13 +17,16 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "oj", ">= 3.0"
+  spec.add_dependency "qonfig", "~> 0.24"
+
   spec.add_development_dependency "activesupport", "~> 5.2.0"
   spec.add_development_dependency "bundler", ">= 1.15"
   spec.add_development_dependency "fuubar", "~> 2.0"
@@ -32,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "armitage-rubocop", "0.82"
 end

data/lib/crawler_detect.rb

@@ -1,24 +1,46 @@
 # frozen_string_literal: true
 
-require "crawler_detect/detector"
-require "crawler_detect/library"
-require "crawler_detect/library/crawlers"
-require "crawler_detect/library/exclusions"
-require "crawler_detect/library/headers"
-require "crawler_detect/version"
+require "oj"
+require "qonfig"
 
-require "rack/crawler_detect"
+require_relative "crawler_detect/config"
+require_relative "crawler_detect/detector"
+require_relative "crawler_detect/library/loader"
+require_relative "crawler_detect/library/crawlers"
+require_relative "crawler_detect/library/exclusions"
+require_relative "crawler_detect/library/headers"
+require_relative "crawler_detect/library"
+require_relative "crawler_detect/version"
+require_relative "rack/crawler_detect"
 
+# @since 0.1.0
 module CrawlerDetect
   class << self
+    # @param user_agent [String] User-agent string to detect
+    # @return [CrawlerDetect::Detector] Instance of detector class
     def new(user_agent)
       detector(user_agent)
     end
 
+    # @param user_agent [String] User-agent string to detect
+    # @return [true, false] Is User-agent a crawler?
     def is_crawler?(user_agent)
       detector(user_agent).is_crawler?
     end
 
+    # @since 1.0.0
+    # @param config [Proc]
+    def setup!(&config)
+      @config = CrawlerDetect::Config.new(&config)
+      Library::DATA_CLASSES.each(&:reload_data)
+    end
+
+    # @since 1.0.0
+    # @return [CrawlerDetect::Config] Instance of configuration class
+    def config
+      @config ||= CrawlerDetect::Config.new
+    end
+
     private
 
     def detector(user_agent)