voight_kampff 0.2.2 → 1.0.0

data/lib/tasks/voight_kampff.rake

@@ -1,12 +1,14 @@
 namespace :voight_kampff do
-
-  desc 'Import a new uesr_agents.yml file'
+  desc 'Import a new crawler-user-agents.json file'
   task :import_user_agents, :url do |t, args|
-    args.with_defaults :url => 'http://www.user-agents.org/allagents.xml'
-    agents = VoightKampff::UserAgentsParser.new(args[:url])
+    args.with_defaults url: 'https://raw.githubusercontent.com/monperrus/crawler-user-agents/master/crawler-user-agents.json'
 
-    file = File.open(Rails.root.join('config','user_agents.yml'), 'w')
-    file.write(agents.to_yaml)
-  end
+    require 'net/http'
 
+    uri = URI(args[:url])
+    contents = Net::HTTP.get(uri)
+
+    file = File.open('./config/crawler-user-agents.json', 'w')
+    file.write(contents)
+  end
 end

data/lib/voight_kampff.rb

@@ -1,11 +1,11 @@
+require 'json'
+
 require 'voight_kampff/test'
-require 'voight_kampff/user_agents_parser'
+require 'voight_kampff/rack_request'
 require 'voight_kampff/engine' if defined?(Rails)
 
 module VoightKampff
-
   class << self
-
     def root
       require 'pathname'
       Pathname.new File.expand_path '..', File.dirname(__FILE__)
@@ -20,36 +20,10 @@ module VoightKampff
     end
     alias :replicant? :bot?
 
-    def browser?(user_agent_string)
-      test(user_agent_string).browser?
-    end
-
-    def checker?(user_agent_string)
-      test(user_agent_string).checker?
-    end
-
-    def downloader?(user_agent_string)
-      test(user_agent_string).downloader?
-    end
-
-    def proxy?(user_agent_string)
-      test(user_agent_string).proxy?
-    end
-
-    def crawler?(user_agent_string)
-      test(user_agent_string).crawler?
-    end
-
-    def spam?(user_agent_string)
-      test(user_agent_string).spam?
-    end
-
     private
 
     def test(user_agent_string)
       VoightKampff::Test.new(user_agent_string)
     end
-
   end
-
 end

data/lib/voight_kampff/engine.rb

@@ -1,12 +1,7 @@
-require 'voight_kampff'
-require 'rails'
-
 module VoightKampff
   class Engine < Rails::Engine
-  
     rake_tasks do
       load 'tasks/voight_kampff.rake'
     end
-  
   end
 end

data/lib/voight_kampff/rack_request.rb

@@ -0,0 +1,11 @@
+# Reopen the Rack::Request class to add bot detection methods
+class Rack::Request
+  def human?
+    VoightKampff::Test.new(user_agent).human?
+  end
+
+  def bot?
+    VoightKampff::Test.new(user_agent).bot?
+  end
+  alias :replicant? :bot?
+end

data/lib/voight_kampff/test.rb

@@ -1,88 +1,47 @@
 module VoightKampff
-
   class Test
+    CRAWLERS_FILENAME = 'crawler-user-agents.json'
 
     attr_accessor :user_agent_string
-    attr_accessor :types
 
     def initialize(user_agent_string)
-      load_agents
       @user_agent_string = user_agent_string
-      @types = Array(agent['types']).map(&:to_sym)
-      @description = agent['description']
     end
 
     def agent
-      @agent ||= @@agents.find do |agent|
-        if agent['string_match'] == 'regex'
-          @user_agent_string =~ Regexp.new(agent['string'], Regexp::IGNORECASE)
-        else
-          @user_agent_string.to_s.casecmp(agent['string'].to_s) == 0
-        end
-      end || {}
-    end
+      load_crawlers
 
-    def has_type?(*types)
-      return nil if agent.empty?
-
-      types.any? do |type|
-        @types.include? type
-      end
+      @agent ||= @@crawlers.find do |crawler|
+        user_agent_string =~ Regexp.new(crawler['pattern'], Regexp::IGNORECASE)
+      end || {}
     end
 
     def human?
-      has_type? :browser, :downloader, :proxy
+      agent.empty?
     end
 
     def bot?
-      has_type? :checker, :crawler, :spam
+      !human?
     end
     alias :replicant? :bot?
 
-    def browser?
-      has_type? :browser
-    end
-
-    def checker?
-      has_type? :checker
-    end
-
-    def downloader?
-      has_type? :downloader
-    end
+    private
 
-    def proxy?
-      has_type? :proxy
-    end
+    def lookup_paths
+      # These paths should be orderd by priority
+      base_paths = []
+      base_paths << Rails.root if defined? Rails
+      base_paths << VoightKampff.root
 
-    def crawler?
-      has_type? :crawler
+      base_paths.map { |p| p.join('config', CRAWLERS_FILENAME) }
     end
 
-    def spam?
-      has_type? :spam
+    def preferred_path
+      lookup_paths.find { |path| File.exists? path }
     end
 
-    private
-
-    def load_agents
-      @@agents ||= []
-      if @@agents.empty?
-
-        base_paths = []
-        base_paths << Rails.root if defined? Rails
-        base_paths << VoightKampff.root
-        rel_path = ['config', 'user_agents.yml']
-
-        base_paths.any? do |base_path|
-          if File.exists? base_path.join(*rel_path)
-            @@agents = YAML.load(File.open(base_path.join(*rel_path), 'r'))
-          end
-        end
-
-      end
+    def load_crawlers
+      @@crawlers ||= JSON.load(File.open(preferred_path, 'r'))
     end
-
   end
-
 end

data/lib/voight_kampff/version.rb

@@ -1,3 +1,3 @@
 module VoightKampff
-  VERSION = '0.2.2'
+  VERSION = '1.0.0'
 end

data/spec/controllers/replicants_controller_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe ReplicantsController, type: :controller do
+  let(:user_agent_string) { '' }
+  before do
+    expect(request).to receive(:user_agent).and_return user_agent_string
+    get :index
+  end
+
+  HUMANS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is forbidden' do
+        expect(response.status).to eql 403
+        expect(response.body).to match(/No replicants here/)
+      end
+    end
+  end
+
+  REPLICANTS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is successful' do
+        expect(response.status).to eql 200
+        expect(response.body).to match(/Rick Deckard/)
+      end
+    end
+  end
+end

data/spec/internal/app/controllers/replicants_controller.rb

@@ -0,0 +1,13 @@
+class ReplicantsController < ActionController::Base
+  def index
+    header = "Replicants:\n===========\n"
+
+    status, content = if request.bot?
+      [200, '- Rick Deckard']
+    else
+      [403, 'No replicants here']
+    end
+
+    render plain: header + content, status: status
+  end
+end

data/spec/internal/config/routes.rb

@@ -0,0 +1,4 @@
+Rails.application.routes.draw do
+  resources :replicants, only: :index
+  root to: 'replicants#index'
+end

data/spec/internal/log/.gitignore

@@ -0,0 +1 @@
+*.log

data/spec/lib/voight_kampff/rack_request_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Rack::Request do
+  let(:user_agent_string) { }
+  let(:env) { {'HTTP_USER_AGENT' => user_agent_string} }
+  subject { Rack::Request.new(env) }
+
+  it { expect(subject).to respond_to :human? }
+  it { expect(subject).to respond_to :bot? }
+  it { expect(subject).to respond_to :replicant? }
+
+  HUMANS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is not a replicant' do
+        expect(subject.human?).to eql true
+        expect(subject.bot?).to eql false
+      end
+    end
+  end
+
+  REPLICANTS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is a replicant' do
+        expect(subject.bot?).to eql true
+        expect(subject.human?).to eql false
+      end
+    end
+  end
+end

data/spec/lib/voight_kampff/test_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe VoightKampff::Test do
+  let(:user_agent_string) { nil }
+  subject { VoightKampff::Test.new(user_agent_string) }
+
+  HUMANS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is not a replicant' do
+        expect(subject.human?).to be true
+        expect(subject.bot?).to be false
+      end
+    end
+  end
+
+  REPLICANTS.each do |name, ua_string|
+    context "when user agent is #{name}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is a replicant' do
+        expect(subject.bot?).to be true
+        expect(subject.human?).to be false
+      end
+    end
+  end
+end

data/spec/lib/voight_kampff_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe VoightKampff do
+  subject { VoightKampff }
+
+  HUMANS.each do |name, ua_string|
+    context "when user agent is #{ua_string}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is not a replicant' do
+        expect(subject.human?(user_agent_string)).to be true
+        expect(subject.bot?(user_agent_string)).to be false
+      end
+    end
+  end
+
+  REPLICANTS.each do |name, ua_string|
+    context "when user agent is #{ua_string}" do
+      let(:user_agent_string) { ua_string }
+
+      it 'is a replicant' do
+        expect(subject.bot?(user_agent_string)).to be true
+        expect(subject.human?(user_agent_string)).to be false
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require 'bundler/setup'
+require 'combustion'
+require 'voight_kampff'
+
+Combustion.initialize! :action_controller
+
+require 'rspec/rails'
+Dir['./spec/support/**/*.rb'].each { |f| require f }
+
+RSpec.configure do |config|
+end

data/spec/support/humans.rb

@@ -0,0 +1,9 @@
+HUMANS = {
+  'Unknown' => nil, # for the moment we're treating a blank user agent string as not a bot
+  'Chrome' => 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36',
+  'Firefox' => 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1',
+  'Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A',
+  'Internet Explorer' => 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko',
+  'Chrome Mobile' => 'Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19',
+  'Safari for iOS' => 'Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53'
+}

data/spec/support/replicants.rb

@@ -0,0 +1,5 @@
+REPLICANTS = {
+  'Googlebot' => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+  'Bingbot' => 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
+  'Yahoo! Slurp' => 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)'
+}

data/voight_kampff.gemspec

@@ -21,10 +21,9 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {tests}/**/*`.split("\n")
   s.require_path  = 'lib'
 
-  s.add_dependency 'httpclient', '~> 2.2'
+  s.add_dependency 'rack', '~> 1.6'
 
-  # Nokogiri 1.6.0 has dropped support for Ruby 1.8.7
-  s.add_dependency 'nokogiri', (RUBY_VERSION.match("1.8.7") ? '1.5.10' : '~> 1.6')
-  s.add_development_dependency 'minitest', '~> 4.7'
-  s.add_development_dependency 'rdoc', '~> 2.4'
+  s.add_development_dependency 'rails', '~> 4.2'
+  s.add_development_dependency 'rspec-rails', '~> 3.3'
+  s.add_development_dependency 'combustion', '~> 0.5'
 end

metadata

@@ -1,141 +1,101 @@
 --- !ruby/object:Gem::Specification
 name: voight_kampff
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Adam Crownoble
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-05 00:00:00.000000000 Z
+date: 2015-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: httpclient
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :runtime
+        version: '4.2'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '4.2'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: combustion
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '0.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '0.5'
 description: Voight-Kampff detects bots, spiders, crawlers and replicants
 email: adam@obledesign.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".autotest"
 - ".gitignore"
 - ".ruby-version"
-- CHANGELOG.rdoc
 - Gemfile
 - MIT-LICENSE
-- README.markdown
-- Rakefile
-- config/initializers/extend_action_dispatch_request.rb
-- config/user_agents.yml
+- README.md
+- config.ru
+- config/crawler-user-agents.json
 - lib/tasks/voight_kampff.rake
 - lib/voight_kampff.rb
-- lib/voight_kampff/base.rb
 - lib/voight_kampff/engine.rb
+- lib/voight_kampff/rack_request.rb
 - lib/voight_kampff/test.rb
-- lib/voight_kampff/user_agents_parser.rb
 - lib/voight_kampff/version.rb
-- tests/spec/minitest_helper.rb
-- tests/spec/voight_kampff_spec.rb
-- tests/test_app/.gitignore
-- tests/test_app/Gemfile
-- tests/test_app/Gemfile.lock
-- tests/test_app/Rakefile
-- tests/test_app/app/controllers/application_controller.rb
-- tests/test_app/app/helpers/application_helper.rb
-- tests/test_app/app/mailers/.gitkeep
-- tests/test_app/app/models/.gitkeep
-- tests/test_app/app/views/layouts/application.html.erb
-- tests/test_app/config.ru
-- tests/test_app/config/application.rb
-- tests/test_app/config/boot.rb
-- tests/test_app/config/database.yml
-- tests/test_app/config/environment.rb
-- tests/test_app/config/environments/development.rb
-- tests/test_app/config/environments/production.rb
-- tests/test_app/config/environments/test.rb
-- tests/test_app/config/initializers/backtrace_silencers.rb
-- tests/test_app/config/initializers/inflections.rb
-- tests/test_app/config/initializers/mime_types.rb
-- tests/test_app/config/initializers/secret_token.rb
-- tests/test_app/config/initializers/session_store.rb
-- tests/test_app/config/initializers/wrap_parameters.rb
-- tests/test_app/config/locales/en.yml
-- tests/test_app/config/routes.rb
-- tests/test_app/doc/README_FOR_APP
-- tests/test_app/lib/assets/.gitkeep
-- tests/test_app/lib/tasks/.gitkeep
-- tests/test_app/log/.gitkeep
-- tests/test_app/public/404.html
-- tests/test_app/public/422.html
-- tests/test_app/public/500.html
-- tests/test_app/public/favicon.ico
-- tests/test_app/public/index.html
-- tests/test_app/public/robots.txt
-- tests/test_app/script/rails
-- tests/test_app/test/fixtures/.gitkeep
-- tests/test_app/test/functional/.gitkeep
-- tests/test_app/test/integration/.gitkeep
-- tests/test_app/test/performance/browsing_test.rb
-- tests/test_app/test/test_helper.rb
-- tests/test_app/test/unit/.gitkeep
-- tests/test_app/vendor/assets/javascripts/.gitkeep
-- tests/test_app/vendor/assets/stylesheets/.gitkeep
-- tests/test_app/vendor/plugins/.gitkeep
+- spec/controllers/replicants_controller_spec.rb
+- spec/internal/app/controllers/replicants_controller.rb
+- spec/internal/config/routes.rb
+- spec/internal/log/.gitignore
+- spec/internal/public/favicon.ico
+- spec/lib/voight_kampff/rack_request_spec.rb
+- spec/lib/voight_kampff/test_spec.rb
+- spec/lib/voight_kampff_spec.rb
+- spec/spec_helper.rb
+- spec/support/humans.rb
+- spec/support/replicants.rb
 - voight_kampff.gemspec
 homepage: https://github.com/biola/Voight-Kampff
 licenses: