crawler_detect 0.1.10 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 342d96d8f386511a05e7ab0894faefca3ea74a60910f378c2b5c1ab26bd530a8
-  data.tar.gz: 5fb6e1759b67aeb2e2d644d6c48141ad02c3c74663a31468ae2f4c53b69df0fe
+  metadata.gz: c0dcfbc40c645fb77a672c9c455c5e306c75ea3d7086f6d9d1f67fc15172ca0d
+  data.tar.gz: fb32a563f4ec5790ff3d2ae68b6f6a8883a48ad383e6caec529b09355857b1f1
 SHA512:
-  metadata.gz: 9ed795dd180037e8a131bc05b3a176488b70601373daabbd459a0f3c21bc49a240eaac4518148c50dc14bd5edd46f92f62e70453c2cbfd0d85002e3cdfc9dbdc
-  data.tar.gz: 1e30cad0d3588db6a89fb95b04e5e09166a9742acc33f14c2dce2d56c25b5578f12e3c572dc2c1401fccf5d526518b851ce41f79e0a3b00f9ef6b361718ad14f
+  metadata.gz: 226d14061717db016ff02f28c5aff050b187a051caf2492d28297e01ee3409d6479683c8e31ed0a8f7235f8f5aadfea938e5f9266e3ead1ec2bafc3a2f7be54b
+  data.tar.gz: db86877f9680fe96f2cf9847f4ecab75fb65117d0d52992e901b25c2a6ac18c1a088e37429fe08b246b7e51df0ad65dd6b4fd0a2bf9e3f77d8d96133d63f359a

data/.rubocop.yml

@@ -1,174 +1,19 @@
-AllCops:
-  TargetRubyVersion: 2.2
-  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
-  # to ignore them, so only the ones explicitly set in this file are enabled.
-  DisabledByDefault: true
-  Exclude:
-  - '**/templates/**/*'
-  - '**/vendor/**/*'
-  - '**/vendor/**/.*'
-  - '**/node_modules/**/*'
-  - 'actionpack/lib/action_dispatch/journey/parser.rb'
+inherit_gem:
+  armitage-rubocop:
+    - lib/rubocop.general.yml
+    - lib/rubocop.rake.yml
+    - lib/rubocop.rspec.yml
 
-# Prefer assert_not_x over refute_x
-CustomCops/RefuteNot:
+AllCops:
+  TargetRubyVersion: 2.7.1
   Include:
-  - '**/test/**/*'
-
-# Prefer &&/|| over and/or.
-Style/AndOr:
-  Enabled: true
-
-# Do not use braces for hash literals when they are the last argument of a
-# method call.
-Style/BracesAroundHashParameters:
-  Enabled: true
-  EnforcedStyle: context_dependent
-
-# Align `when` with `case`.
-Layout/CaseIndentation:
-  Enabled: true
-
-# Align comments with method definitions.
-Layout/CommentIndentation:
-  Enabled: true
-
-Layout/ElseAlignment:
-  Enabled: true
-
-# Align `end` with the matching keyword or starting expression except for
-# assignments, where it should be aligned with the LHS.
-Layout/EndAlignment:
-  Enabled: true
-  EnforcedStyleAlignWith: variable
-  AutoCorrect: true
-
-Layout/EmptyLineAfterMagicComment:
-  Enabled: true
-
-# In a regular class definition, no empty lines around the body.
-Layout/EmptyLinesAroundClassBody:
-  Enabled: true
-
-# In a regular method definition, no empty lines around the body.
-Layout/EmptyLinesAroundMethodBody:
-  Enabled: true
+    - lib/**/*.rb
+    - spec/**/*.rb
+    - Gemfile
+    - Rakefile
+    - crawler_detect.gemspec
+    - bin/console
 
-# In a regular module definition, no empty lines around the body.
-Layout/EmptyLinesAroundModuleBody:
-  Enabled: true
-
-Layout/FirstParameterIndentation:
-  Enabled: true
-
-# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
-Style/HashSyntax:
-  Enabled: true
-
-# Method definitions after `private` or `protected` isolated calls need one
-# extra level of indentation.
-Layout/IndentationConsistency:
-  Enabled: true
-  EnforcedStyle: rails
-
-# Two spaces, no tabs (for indentation).
-Layout/IndentationWidth:
-  Enabled: true
-
-Layout/LeadingCommentSpace:
-  Enabled: true
-
-Layout/SpaceAfterColon:
-  Enabled: true
-
-Layout/SpaceAfterComma:
-  Enabled: true
-
-Layout/SpaceAroundEqualsInParameterDefault:
-  Enabled: true
-
-Layout/SpaceAroundKeyword:
-  Enabled: true
-
-Layout/SpaceAroundOperators:
-  Enabled: true
-
-Layout/SpaceBeforeComma:
-  Enabled: true
-
-Layout/SpaceBeforeFirstArg:
-  Enabled: true
-
-Style/DefWithParentheses:
-  Enabled: true
-
-# Defining a method with parameters needs parentheses.
-Style/MethodDefParentheses:
-  Enabled: true
-
-Style/FrozenStringLiteralComment:
-  Enabled: true
-  EnforcedStyle: always
-  Exclude:
-  - 'actionview/test/**/*.builder'
-  - 'actionview/test/**/*.ruby'
-  - 'actionpack/test/**/*.builder'
-  - 'actionpack/test/**/*.ruby'
-  - 'activestorage/db/migrate/**/*.rb'
-  - 'db/migrate/**/*.rb'
-  - 'db/*.rb'
-
-# Use `foo {}` not `foo{}`.
-Layout/SpaceBeforeBlockBraces:
-  Enabled: true
-
-# Use `foo { bar }` not `foo {bar}`.
-Layout/SpaceInsideBlockBraces:
-  Enabled: true
-
-# Use `{ a: 1 }` not `{a:1}`.
-Layout/SpaceInsideHashLiteralBraces:
-  Enabled: true
-
-Layout/SpaceInsideParens:
-  Enabled: true
-
-# Check quotes usage according to lint rule below.
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes
-
-# Detect hard tabs, no hard tabs.
-Layout/Tab:
-  Enabled: true
-
-# Blank lines should not have any spaces.
-Layout/TrailingBlankLines:
-  Enabled: true
-
-# No trailing whitespace.
-Layout/TrailingWhitespace:
-  Enabled: true
-
-# Use quotes for string literals when they are enough.
-Style/UnneededPercentQ:
-  Enabled: true
-
-# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
-Lint/RequireParentheses:
-  Enabled: true
-
-Lint/StringConversionInInterpolation:
-  Enabled: true
-
-Style/RedundantReturn:
-  Enabled: true
-  AllowMultipleReturnValues: true
-
-Style/Semicolon:
-  Enabled: true
-  AllowAsExpressionSeparator: true
-
-# Prefer Foo.method over Foo::method
-Style/ColonMethodCall:
-  Enabled: true

data/.travis.yml

@@ -1,16 +1,26 @@
----
-sudo: false
+os: linux
+dist: xenial
 language: ruby
 cache: bundler
+before_install:
+  - gem install bundler
 env:
   - "TEST_GROUP=1"
   - "TEST_GROUP=2"
   - "TEST_GROUP=3"
   - "TEST_GROUP=4"
 rvm:
-  - 2.3
   - 2.5
   - 2.6
-before_install: gem install bundler
-script:
-  - bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+  - 2.7
+stages:
+  - lint
+  - test
+script: bundle exec parallel_rspec spec/ -n 4 --only-group $TEST_GROUP --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+jobs:
+  fast_finish: true
+  include:
+    - stage: lint
+      rvm: 2.7
+      env: "TEST_GROUP=none"
+      script: bundle exec rubocop

data/CHANGELOG.md

@@ -0,0 +1,32 @@
+# CrawlerDetect major changes
+
+This changelog **does not contain** raw data updates 
+but only major changes.
+
+1.0.0
+---------
+- Use raw JSON files instead of copy them to rb [#8]
+- Add CrawlerDetect::Config to make it possible to have own raw files [#8]
+- Add bin/update_raw_files to update raw files from PHP lib [#8] 
+- Add Changelog
+
+0.1.11
+---------
+- Add thread safety [#19]
+
+0.1.6
+---------
+- Strip crawler name [#10]
+
+0.1.2
+---------
+- Add parallel tests [#2]
+
+0.1.1
+---------
+- Fix: rack request
+
+0.1.0
+---------
+- init
+

data/Gemfile.lock

@@ -0,0 +1,119 @@
+PATH
+  remote: .
+  specs:
+    crawler_detect (1.0.1)
+      oj (>= 3.0)
+      qonfig (~> 0.24)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    armitage-rubocop (0.82.0)
+      rubocop (= 0.82.0)
+      rubocop-performance (= 1.5.2)
+      rubocop-rails (= 2.5.2)
+      rubocop-rake (= 0.5.1)
+      rubocop-rspec (= 1.38.1)
+    ast (2.4.0)
+    awesome_print (1.8.0)
+    byebug (11.1.2)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.6)
+    diff-lcs (1.3)
+    fuubar (2.5.0)
+      rspec-core (~> 3.0)
+      ruby-progressbar (~> 1.4)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.4)
+    method_source (1.0.0)
+    minitest (5.14.1)
+    oj (3.10.6)
+    parallel (1.19.1)
+    parallel_tests (2.32.0)
+      parallel
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    pry-meta (0.0.10)
+      awesome_print
+      pry
+      pry-byebug
+      pry-remote
+    pry-remote (0.1.8)
+      pry (~> 0.9)
+      slop (~> 3.0)
+    qonfig (0.24.1)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.82.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.5.2)
+      activesupport
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (1.38.1)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    slop (3.6.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 6.0.3)
+  armitage-rubocop (= 0.82)
+  bundler (>= 1.15)
+  crawler_detect!
+  fuubar (~> 2.0)
+  parallel_tests (~> 2.0)
+  pry-meta (~> 0.0.10)
+  rack-test (~> 1.1)
+  rake (>= 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -54,5 +54,18 @@ end
 ```
 use Rack::CrawlerDetect
 ```
+## Configuration
+In some cases you may want to use your own white-list, or black-list or list of http-headers to detect User-agent.
+
+It is possible to do via `CrawlerDetect::Config`. For example, you may have initializer like this:
+```
+CrawlerDetect.setup! do |config|
+  config.raw_headers_path    = File.expand_path("crawlers/MyHeaders.json", __dir__)
+  config.raw_crawlers_path   = File.expand_path("crawlers/MyCrawlers.json", __dir__)
+  config.raw_exclusions_path = File.expand_path("crawlers/MyExclusions.json", __dir__)
+end
+```
+Make sure that your files are correct JSON files. 
+Look at [the raw files](https://github.com/loadkpi/crawler_detect/tree/master/lib/crawler_detect/library/raw) which are used by default for more information. 
 ## License
 MIT License

data/bin/update_raw_files

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
+wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
+
+wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
+wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt

data/crawler_detect.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "crawler_detect/version"
 
@@ -17,14 +17,17 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path("..", __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "activesupport", "~> 5.2.0"
+  spec.add_dependency "oj", ">= 3.0"
+  spec.add_dependency "qonfig", "~> 0.24"
+
+  spec.add_development_dependency "activesupport", "~> 6.0.3"
   spec.add_development_dependency "bundler", ">= 1.15"
   spec.add_development_dependency "fuubar", "~> 2.0"
   spec.add_development_dependency "parallel_tests", "~> 2.0"
@@ -32,4 +35,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rack-test", "~> 1.1"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "armitage-rubocop", "0.82"
 end