crawler_detect 0.1.11 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97314f6307705c0123a25333340c7509138cd8991eb1946a52d66c99df08ddac
-  data.tar.gz: 135a67b8502348dde1c86dd0e17e894dceafeb61b8acb74487f3d6e1832954e0
+  metadata.gz: 6fed2e5d51ebe5a84f9efd591c8ec11eeb7b8ed6d73c06680972482040fbb2e3
+  data.tar.gz: 90c32dbab1d592fcb95641f06f995454dace41148c30ca7006082c57c0eeca79
 SHA512:
-  metadata.gz: 1f6dc6ef31c358ce0a5e94469971021ba5a1e1c8ba9133836e839053e4717c32d94a0c3dbb2d828ada989192f70e85d899ad87b9b56f0e26ef33f08d45209112
-  data.tar.gz: ed8cf6c609fe3b7e3102d9a6d60358d9e80e19f69b1a8c4b218bc929642991389e40fecf48c5de69975d76373b1406b7a89e1c9ec5ccdf2dd727710eb3363524
+  metadata.gz: 13a526e036c48f35e70e6d82a1270d934dc741e125ca106ce290fae1adf8e207e001b3f1d4a240a8778bdc55f45e2d929973fb8e134edeeb7be37dfb62971acd
+  data.tar.gz: af479ed4af6732062abe1ae6678d6a8f3afd38c1ae1231ceef7bd97d8ee5ee3cf0f4cd8aabaf17b61860410a9f3d9f93633c26f1f7084527380bb77ef1e71e86

data/.rubocop.yml

@@ -1,174 +1,19 @@
-AllCops:
-  TargetRubyVersion: 2.2
-  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
-  # to ignore them, so only the ones explicitly set in this file are enabled.
-  DisabledByDefault: true
-  Exclude:
-  - '**/templates/**/*'
-  - '**/vendor/**/*'
-  - '**/vendor/**/.*'
-  - '**/node_modules/**/*'
-  - 'actionpack/lib/action_dispatch/journey/parser.rb'
+inherit_gem:
+  armitage-rubocop:
+    - lib/rubocop.general.yml
+    - lib/rubocop.rake.yml
+    - lib/rubocop.rspec.yml
 
-# Prefer assert_not_x over refute_x
-CustomCops/RefuteNot:
+AllCops:
+  TargetRubyVersion: 2.7.1
   Include:
-  - '**/test/**/*'
-
-# Prefer &&/|| over and/or.
-Style/AndOr:
-  Enabled: true
-
-# Do not use braces for hash literals when they are the last argument of a
-# method call.
-Style/BracesAroundHashParameters:
-  Enabled: true
-  EnforcedStyle: context_dependent
-
-# Align `when` with `case`.
-Layout/CaseIndentation:
-  Enabled: true
-
-# Align comments with method definitions.
-Layout/CommentIndentation:
-  Enabled: true
-
-Layout/ElseAlignment:
-  Enabled: true
-
-# Align `end` with the matching keyword or starting expression except for
-# assignments, where it should be aligned with the LHS.
-Layout/EndAlignment:
-  Enabled: true
-  EnforcedStyleAlignWith: variable
-  AutoCorrect: true
-
-Layout/EmptyLineAfterMagicComment:
-  Enabled: true
-
-# In a regular class definition, no empty lines around the body.
-Layout/EmptyLinesAroundClassBody:
-  Enabled: true
-
-# In a regular method definition, no empty lines around the body.
-Layout/EmptyLinesAroundMethodBody:
-  Enabled: true
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - crawler_detect.gemspec
+    - bin/console
 
-# In a regular module definition, no empty lines around the body.
-Layout/EmptyLinesAroundModuleBody:
-  Enabled: true
-
-Layout/FirstParameterIndentation:
-  Enabled: true
-
-# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
-Style/HashSyntax:
-  Enabled: true
-
-# Method definitions after `private` or `protected` isolated calls need one
-# extra level of indentation.
-Layout/IndentationConsistency:
-  Enabled: true
-  EnforcedStyle: rails
-
-# Two spaces, no tabs (for indentation).
-Layout/IndentationWidth:
-  Enabled: true
-
-Layout/LeadingCommentSpace:
-  Enabled: true
-
-Layout/SpaceAfterColon:
-  Enabled: true
-
-Layout/SpaceAfterComma:
-  Enabled: true
-
-Layout/SpaceAroundEqualsInParameterDefault:
-  Enabled: true
-
-Layout/SpaceAroundKeyword:
-  Enabled: true
-
-Layout/SpaceAroundOperators:
-  Enabled: true
-
-Layout/SpaceBeforeComma:
-  Enabled: true
-
-Layout/SpaceBeforeFirstArg:
-  Enabled: true
-
-Style/DefWithParentheses:
-  Enabled: true
-
-# Defining a method with parameters needs parentheses.
-Style/MethodDefParentheses:
-  Enabled: true
-
-Style/FrozenStringLiteralComment:
-  Enabled: true
-  EnforcedStyle: always
-  Exclude:
-  - 'actionview/test/**/*.builder'
-  - 'actionview/test/**/*.ruby'
-  - 'actionpack/test/**/*.builder'
-  - 'actionpack/test/**/*.ruby'
-  - 'activestorage/db/migrate/**/*.rb'
-  - 'db/migrate/**/*.rb'
-  - 'db/*.rb'
-
-# Use `foo {}` not `foo{}`.
-Layout/SpaceBeforeBlockBraces:
-  Enabled: true
-
-# Use `foo { bar }` not `foo {bar}`.
-Layout/SpaceInsideBlockBraces:
-  Enabled: true
-
-# Use `{ a: 1 }` not `{a:1}`.
-Layout/SpaceInsideHashLiteralBraces:
-  Enabled: true
-
-Layout/SpaceInsideParens:
-  Enabled: true
-
-# Check quotes usage according to lint rule below.
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
-
-# Detect hard tabs, no hard tabs.
-Layout/Tab:
-  Enabled: true
-
-# Blank lines should not have any spaces.
-Layout/TrailingBlankLines:
-  Enabled: true
-
-# No trailing whitespace.
-Layout/TrailingWhitespace:
-  Enabled: true
-
-# Use quotes for string literals when they are enough.
-Style/UnneededPercentQ:
-  Enabled: true
-
-# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
-Lint/RequireParentheses:
-  Enabled: true
-
-Lint/StringConversionInInterpolation:
-  Enabled: true
-
-Style/RedundantReturn:
-  Enabled: true
-  AllowMultipleReturnValues: true
-
-Style/Semicolon:
-  Enabled: true
-  AllowAsExpressionSeparator: true
-
-# Prefer Foo.method over Foo::method
-Style/ColonMethodCall:
-  Enabled: true

data/.travis.yml

@@ -1,16 +1,26 @@
----
-sudo: false
+os: linux
+dist: xenial
 language: ruby
 cache: bundler
+before_install:
+  - gem install bundler
 env:
   - "TEST_GROUP=1"
   - "TEST_GROUP=2"
   - "TEST_GROUP=3"
   - "TEST_GROUP=4"
 rvm:
-  - 2.3
   - 2.5
   - 2.6
-before_install: gem install bundler
-script:
-  - bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+  - 2.7
+stages:
+  - lint
+  - test
+script: bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+jobs:
+  fast_finish: true
+  include:
+    - stage: lint
+      rvm: 2.7
+      env: "TEST_GROUP=none"
+      script: bundle exec rubocop

data/CHANGELOG.md

@@ -0,0 +1,32 @@
+# CrawlerDetect major changes
+
+This changelog **does not contain** raw data updates 
+but only major changes.
+
+1.0.0
+---------
+- Use raw JSON files instead of copy them to rb [#8]
+- Add CrawlerDetect::Config to make it possible to have own raw files [#8]
+- Add bin/update_raw_files to update raw files from PHP lib [#8] 
+- Add Changelog
+
+0.1.11
+---------
+- Add thread safety [#19]
+
+0.1.6
+---------
+- Strip crawler name [#10]
+
+0.1.2
+---------
+- Add parallel tests [#2]
+
+0.1.1
+---------
+- Fix: rack request
+
+0.1.0
+---------
+- init
+

data/Gemfile.lock

@@ -0,0 +1,117 @@
+PATH
+  remote: .
+  specs:
+    crawler_detect (1.0.2)
+      qonfig (~> 0.24)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    armitage-rubocop (0.82.0)
+      rubocop (= 0.82.0)
+      rubocop-performance (= 1.5.2)
+      rubocop-rails (= 2.5.2)
+      rubocop-rake (= 0.5.1)
+      rubocop-rspec (= 1.38.1)
+    ast (2.4.0)
+    awesome_print (1.8.0)
+    byebug (11.1.2)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.6)
+    diff-lcs (1.3)
+    fuubar (2.5.0)
+      rspec-core (~> 3.0)
+      ruby-progressbar (~> 1.4)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
+    method_source (1.0.0)
+    minitest (5.14.1)
+    parallel (1.19.1)
+    parallel_tests (2.32.0)
+      parallel
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    pry-meta (0.0.10)
+      awesome_print
+      pry
+      pry-byebug
+      pry-remote
+    pry-remote (0.1.8)
+      pry (~> 0.9)
+      slop (~> 3.0)
+    qonfig (0.24.1)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.5.2)
+      activesupport
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (1.38.1)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    slop (3.6.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 6.0.3)
+  armitage-rubocop (= 0.82)
+  bundler (>= 1.15)
+  crawler_detect!
+  fuubar (~> 2.0)
+  parallel_tests (~> 2.0)
+  pry-meta (~> 0.0.10)
+  rack-test (~> 1.1)
+  rake (>= 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -54,5 +54,18 @@ end
 ```
 use Rack::CrawlerDetect
 ```
+## Configuration
+In some cases you may want to use your own white-list, or black-list or list of http-headers to detect User-agent.
+
+It is possible to do via `CrawlerDetect::Config`. For example, you may have initializer like this:
+```
+CrawlerDetect.setup! do |config|
+  config.raw_headers_path    = File.expand_path("crawlers/MyHeaders.json", __dir__)
+  config.raw_crawlers_path   = File.expand_path("crawlers/MyCrawlers.json", __dir__)
+  config.raw_exclusions_path = File.expand_path("crawlers/MyExclusions.json", __dir__)
+end
+```
+Make sure that your files are correct JSON files. 
+Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information. 
 ## License
 MIT License

data/bin/update_raw_files

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
+
+wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
+wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt

data/crawler_detect.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "crawler_detect/version"
 
@@ -17,14 +17,16 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "activesupport", "~> 5.2.0"
+  spec.add_dependency "qonfig", "~> 0.24"
+
+  spec.add_development_dependency "activesupport", "~> 6.0.3"
   spec.add_development_dependency "bundler", ">= 1.15"
   spec.add_development_dependency "fuubar", "~> 2.0"
   spec.add_development_dependency "parallel_tests", "~> 2.0"
@@ -32,4 +34,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "armitage-rubocop", "0.82"
 end