crawler_detect 0.0.2

data/lib/crawler_detect/library/exclusions.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module CrawlerDetect
+  module Library
+    module Exclusions
+      EXCLUSIONS = %q[
+Safari.[\d\.]*
+Firefox.[\d\.]*
+Chrome.[\d\.]*
+Chromium.[\d\.]*
+MSIE.[\d\.]
+Opera\/[\d\.]*
+Mozilla.[\d\.]*
+AppleWebKit.[\d\.]*
+Trident.[\d\.]*
+Windows NT.[\d\.]*
+Android [\d\.]*
+Macintosh.
+Ubuntu
+Linux
+[ ]Intel
+Mac OS X [\d_]*
+(like )?Gecko(.[\d\.]*)?
+KHTML,
+CriOS.[\d\.]*
+CPU iPhone OS ([0-9_])* like Mac OS X
+CPU OS ([0-9_])* like Mac OS X
+iPod
+compatible
+x86_..
+i686
+x64
+X11
+rv:[\d\.]*
+Version.[\d\.]*
+WOW64
+Win64
+Dalvik.[\d\.]*
+ \.NET CLR [\d\.]*
+Presto.[\d\.]*
+Media Center PC
+BlackBerry
+Build
+Opera Mini\/\d{1,2}\.\d{1,2}\.[\d\.]*\/\d{1,2}\.
+Opera
+ \.NET[\d\.]*
+cubot
+; M bot
+; B bot
+; IDbot
+; ID bot
+; POWER BOT
+;
+      ].strip.split(/\n+/).freeze
+    end
+  end
+end

data/lib/crawler_detect/library/headers.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module CrawlerDetect
+  module Library
+    module Headers
+      HEADERS = [
+        # The default User-Agent string.
+        "HTTP_USER_AGENT",
+        # Header can occur on devices using Opera Mini.
+        "HTTP_X_OPERAMINI_PHONE_UA",
+        # Vodafone specific header: http://www.seoprinciple.com/mobile-web-community-still-angry-at-vodafone/24/
+        "HTTP_X_DEVICE_USER_AGENT",
+        "HTTP_X_ORIGINAL_USER_AGENT",
+        "HTTP_X_SKYFIRE_PHONE",
+        "HTTP_X_BOLT_PHONE_UA",
+        "HTTP_DEVICE_STOCK_UA",
+        "HTTP_X_UCBROWSER_DEVICE_UA",
+        # Sometimes, bots (especially Google) use a genuine user agent, but fill this header in with their email address
+        "HTTP_FROM",
+        # Seen in use by Netsparker
+        "HTTP_X_SCANNER",
+      ].freeze
+    end
+  end
+end

data/lib/crawler_detect/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module CrawlerDetect
+  VERSION = "0.0.2"
+end

data/lib/rack/crawler_detect.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Rack
+  class CrawlerDetect
+    def initialize(app, options = {})
+      Rack::Request::Helpers.module_eval do
+        def is_crawler?
+          env["rack.crawler_detect"][:is_crawler]
+        end
+
+        def crawler_name
+          env["rack.crawler_detect"][:crawler_name]
+        end
+      end
+      @app = app
+    end
+
+    def call(env)
+      @env = env
+      set_env_variables!
+      @app.call(@env)
+    end
+
+    private
+
+      def set_env_variables!
+        return @env unless user_agent
+        detector = ::CrawlerDetect::Detector.new(user_agent)
+        @env["rack.crawler_detect"] = {
+          is_crawler:   detector.is_crawler?,
+          crawler_name: detector.crawler_name,
+        }
+      end
+
+      def user_agent
+        @user_agent ||= begin
+          user_agent_headers.map do |header|
+            @env[header]
+          end.compact.join(" ")
+        end
+      end
+
+      def user_agent_headers
+        ::CrawlerDetect::Library.get_array("headers")
+      end
+  end
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: crawler_detect
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Pavel Kozlov
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-08-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fuubar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: parallel_tests
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: pry-meta
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.10
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.10
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: CrawlerDetect is a library to detect bots/crawlers via the user agent
+email:
+- loadkpi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- crawler_detect.gemspec
+- lib/crawler_detect.rb
+- lib/crawler_detect/detector.rb
+- lib/crawler_detect/library.rb
+- lib/crawler_detect/library/crawlers.rb
+- lib/crawler_detect/library/exclusions.rb
+- lib/crawler_detect/library/headers.rb
+- lib/crawler_detect/version.rb
+- lib/rack/crawler_detect.rb
+homepage: https://github.com/loadkpi/crawler_detect
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: 'CrawlerDetect: detect bots/crawlers'
+test_files: []