voight_kampff 0.2.2 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -4
- data/README.md +75 -0
- data/config.ru +7 -0
- data/config/crawler-user-agents.json +750 -0
- data/lib/tasks/voight_kampff.rake +9 -7
- data/lib/voight_kampff.rb +3 -29
- data/lib/voight_kampff/engine.rb +0 -5
- data/lib/voight_kampff/rack_request.rb +11 -0
- data/lib/voight_kampff/test.rb +18 -59
- data/lib/voight_kampff/version.rb +1 -1
- data/spec/controllers/replicants_controller_spec.rb +31 -0
- data/spec/internal/app/controllers/replicants_controller.rb +13 -0
- data/spec/internal/config/routes.rb +4 -0
- data/spec/internal/log/.gitignore +1 -0
- data/{tests/test_app → spec/internal}/public/favicon.ico +0 -0
- data/spec/lib/voight_kampff/rack_request_spec.rb +33 -0
- data/spec/lib/voight_kampff/test_spec.rb +28 -0
- data/spec/lib/voight_kampff_spec.rb +27 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/support/humans.rb +9 -0
- data/spec/support/replicants.rb +5 -0
- data/voight_kampff.gemspec +4 -5
- metadata +30 -70
- data/.autotest +0 -22
- data/CHANGELOG.rdoc +0 -16
- data/README.markdown +0 -63
- data/Rakefile +0 -20
- data/config/initializers/extend_action_dispatch_request.rb +0 -46
- data/config/user_agents.yml +0 -12438
- data/lib/voight_kampff/base.rb +0 -7
- data/lib/voight_kampff/user_agents_parser.rb +0 -54
- data/tests/spec/minitest_helper.rb +0 -2
- data/tests/spec/voight_kampff_spec.rb +0 -10
- data/tests/test_app/.gitignore +0 -15
- data/tests/test_app/Gemfile +0 -39
- data/tests/test_app/Rakefile +0 -7
- data/tests/test_app/app/controllers/application_controller.rb +0 -3
- data/tests/test_app/app/helpers/application_helper.rb +0 -2
- data/tests/test_app/app/mailers/.gitkeep +0 -0
- data/tests/test_app/app/models/.gitkeep +0 -0
- data/tests/test_app/app/views/layouts/application.html.erb +0 -14
- data/tests/test_app/config.ru +0 -4
- data/tests/test_app/config/application.rb +0 -62
- data/tests/test_app/config/boot.rb +0 -6
- data/tests/test_app/config/database.yml +0 -25
- data/tests/test_app/config/environment.rb +0 -5
- data/tests/test_app/config/environments/development.rb +0 -37
- data/tests/test_app/config/environments/production.rb +0 -67
- data/tests/test_app/config/environments/test.rb +0 -37
- data/tests/test_app/config/initializers/backtrace_silencers.rb +0 -7
- data/tests/test_app/config/initializers/inflections.rb +0 -15
- data/tests/test_app/config/initializers/mime_types.rb +0 -5
- data/tests/test_app/config/initializers/secret_token.rb +0 -7
- data/tests/test_app/config/initializers/session_store.rb +0 -8
- data/tests/test_app/config/initializers/wrap_parameters.rb +0 -14
- data/tests/test_app/config/locales/en.yml +0 -5
- data/tests/test_app/config/routes.rb +0 -58
- data/tests/test_app/lib/assets/.gitkeep +0 -0
- data/tests/test_app/lib/tasks/.gitkeep +0 -0
- data/tests/test_app/log/.gitkeep +0 -0
- data/tests/test_app/public/404.html +0 -26
- data/tests/test_app/public/422.html +0 -26
- data/tests/test_app/public/500.html +0 -25
- data/tests/test_app/public/index.html +0 -241
- data/tests/test_app/public/robots.txt +0 -5
- data/tests/test_app/script/rails +0 -6
- data/tests/test_app/test/fixtures/.gitkeep +0 -0
- data/tests/test_app/test/functional/.gitkeep +0 -0
- data/tests/test_app/test/integration/.gitkeep +0 -0
- data/tests/test_app/test/performance/browsing_test.rb +0 -12
- data/tests/test_app/test/test_helper.rb +0 -13
- data/tests/test_app/test/unit/.gitkeep +0 -0
- data/tests/test_app/vendor/assets/javascripts/.gitkeep +0 -0
- data/tests/test_app/vendor/assets/stylesheets/.gitkeep +0 -0
- data/tests/test_app/vendor/plugins/.gitkeep +0 -0
data/lib/voight_kampff/base.rb
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
module VoightKampff
|
|
2
|
-
|
|
3
|
-
class UserAgentsParser
|
|
4
|
-
|
|
5
|
-
# To generate a new user_agents.yml file run rake voight_kampff:import_user_agents
|
|
6
|
-
# This class is designed to work with http://www.user-agents.org/allagents.xml
|
|
7
|
-
|
|
8
|
-
require 'nokogiri'
|
|
9
|
-
require 'httpclient'
|
|
10
|
-
|
|
11
|
-
def initialize(url)
|
|
12
|
-
@url = url
|
|
13
|
-
@xml = HTTPClient.new.get_content(url)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def agents
|
|
17
|
-
agents = []
|
|
18
|
-
xml = Nokogiri::XML(@xml)
|
|
19
|
-
xml.xpath('//user-agent').each do |agent|
|
|
20
|
-
agents << {
|
|
21
|
-
'string' => agent.xpath('String').text,
|
|
22
|
-
'string_match' => 'exact', # exact or regex
|
|
23
|
-
'types' => agent.xpath('Type').text.split.map{|t| clarify_type(t) }.compact,
|
|
24
|
-
'description' => agent.xpath('Description').text
|
|
25
|
-
}
|
|
26
|
-
end
|
|
27
|
-
agents
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def to_array
|
|
31
|
-
agents
|
|
32
|
-
end
|
|
33
|
-
alias :to_a :to_array
|
|
34
|
-
|
|
35
|
-
def to_yaml
|
|
36
|
-
agents.to_yaml
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
private
|
|
40
|
-
|
|
41
|
-
def clarify_type(type)
|
|
42
|
-
case type.downcase.to_sym
|
|
43
|
-
when :b then 'browser'
|
|
44
|
-
when :c then 'checker'
|
|
45
|
-
when :d then 'downloader'
|
|
46
|
-
when :p then 'proxy'
|
|
47
|
-
when :r then 'crawler'
|
|
48
|
-
when :s then 'spam'
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
end
|
data/tests/test_app/.gitignore
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# See http://help.github.com/ignore-files/ for more about ignoring files.
|
|
2
|
-
#
|
|
3
|
-
# If you find yourself ignoring temporary files generated by your text editor
|
|
4
|
-
# or operating system, you probably want to add a global ignore instead:
|
|
5
|
-
# git config --global core.excludesfile ~/.gitignore_global
|
|
6
|
-
|
|
7
|
-
# Ignore bundler config
|
|
8
|
-
/.bundle
|
|
9
|
-
|
|
10
|
-
# Ignore the default SQLite database.
|
|
11
|
-
/db/*.sqlite3
|
|
12
|
-
|
|
13
|
-
# Ignore all logfiles and tempfiles.
|
|
14
|
-
/log/*.log
|
|
15
|
-
/tmp
|
data/tests/test_app/Gemfile
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
source 'https://rubygems.org'
|
|
2
|
-
|
|
3
|
-
gem 'rails', '3.2.13'
|
|
4
|
-
|
|
5
|
-
# Bundle edge Rails instead:
|
|
6
|
-
# gem 'rails', :git => 'git://github.com/rails/rails.git'
|
|
7
|
-
|
|
8
|
-
gem 'sqlite3'
|
|
9
|
-
|
|
10
|
-
gem 'voight_kampff', :path => '../../'
|
|
11
|
-
|
|
12
|
-
# Gems used only for assets and not required
|
|
13
|
-
# in production environments by default.
|
|
14
|
-
group :assets do
|
|
15
|
-
gem 'sass-rails', '~> 3.2.3'
|
|
16
|
-
gem 'coffee-rails', '~> 3.2.1'
|
|
17
|
-
|
|
18
|
-
# See https://github.com/sstephenson/execjs#readme for more supported runtimes
|
|
19
|
-
# gem 'therubyracer', :platforms => :ruby
|
|
20
|
-
|
|
21
|
-
gem 'uglifier', '>= 1.0.3'
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
gem 'jquery-rails'
|
|
25
|
-
|
|
26
|
-
# To use ActiveModel has_secure_password
|
|
27
|
-
# gem 'bcrypt-ruby', '~> 3.0.0'
|
|
28
|
-
|
|
29
|
-
# To use Jbuilder templates for JSON
|
|
30
|
-
# gem 'jbuilder'
|
|
31
|
-
|
|
32
|
-
# Use unicorn as the app server
|
|
33
|
-
# gem 'unicorn'
|
|
34
|
-
|
|
35
|
-
# Deploy with Capistrano
|
|
36
|
-
# gem 'capistrano'
|
|
37
|
-
|
|
38
|
-
# To use debugger
|
|
39
|
-
# gem 'debugger'
|
data/tests/test_app/Rakefile
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env rake
|
|
2
|
-
# Add your own tasks in files placed in lib/tasks ending in .rake,
|
|
3
|
-
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
|
|
4
|
-
|
|
5
|
-
require File.expand_path('../config/application', __FILE__)
|
|
6
|
-
|
|
7
|
-
TestApp::Application.load_tasks
|
|
File without changes
|
|
File without changes
|
data/tests/test_app/config.ru
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
require File.expand_path('../boot', __FILE__)
|
|
2
|
-
|
|
3
|
-
require 'rails/all'
|
|
4
|
-
|
|
5
|
-
if defined?(Bundler)
|
|
6
|
-
# If you precompile assets before deploying to production, use this line
|
|
7
|
-
Bundler.require(*Rails.groups(:assets => %w(development test)))
|
|
8
|
-
# If you want your assets lazily compiled in production, use this line
|
|
9
|
-
# Bundler.require(:default, :assets, Rails.env)
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
module TestApp
|
|
13
|
-
class Application < Rails::Application
|
|
14
|
-
# Settings in config/environments/* take precedence over those specified here.
|
|
15
|
-
# Application configuration should go into files in config/initializers
|
|
16
|
-
# -- all .rb files in that directory are automatically loaded.
|
|
17
|
-
|
|
18
|
-
# Custom directories with classes and modules you want to be autoloadable.
|
|
19
|
-
# config.autoload_paths += %W(#{config.root}/extras)
|
|
20
|
-
|
|
21
|
-
# Only load the plugins named here, in the order given (default is alphabetical).
|
|
22
|
-
# :all can be used as a placeholder for all plugins not explicitly named.
|
|
23
|
-
# config.plugins = [ :exception_notification, :ssl_requirement, :all ]
|
|
24
|
-
|
|
25
|
-
# Activate observers that should always be running.
|
|
26
|
-
# config.active_record.observers = :cacher, :garbage_collector, :forum_observer
|
|
27
|
-
|
|
28
|
-
# Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
|
|
29
|
-
# Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
|
|
30
|
-
# config.time_zone = 'Central Time (US & Canada)'
|
|
31
|
-
|
|
32
|
-
# The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
|
|
33
|
-
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
|
|
34
|
-
# config.i18n.default_locale = :de
|
|
35
|
-
|
|
36
|
-
# Configure the default encoding used in templates for Ruby 1.9.
|
|
37
|
-
config.encoding = "utf-8"
|
|
38
|
-
|
|
39
|
-
# Configure sensitive parameters which will be filtered from the log file.
|
|
40
|
-
config.filter_parameters += [:password]
|
|
41
|
-
|
|
42
|
-
# Enable escaping HTML in JSON.
|
|
43
|
-
config.active_support.escape_html_entities_in_json = true
|
|
44
|
-
|
|
45
|
-
# Use SQL instead of Active Record's schema dumper when creating the database.
|
|
46
|
-
# This is necessary if your schema can't be completely dumped by the schema dumper,
|
|
47
|
-
# like if you have constraints or database-specific column types
|
|
48
|
-
# config.active_record.schema_format = :sql
|
|
49
|
-
|
|
50
|
-
# Enforce whitelist mode for mass assignment.
|
|
51
|
-
# This will create an empty whitelist of attributes available for mass-assignment for all models
|
|
52
|
-
# in your app. As such, your models will need to explicitly whitelist or blacklist accessible
|
|
53
|
-
# parameters by using an attr_accessible or attr_protected declaration.
|
|
54
|
-
config.active_record.whitelist_attributes = true
|
|
55
|
-
|
|
56
|
-
# Enable the asset pipeline
|
|
57
|
-
config.assets.enabled = true
|
|
58
|
-
|
|
59
|
-
# Version of your assets, change this if you want to expire all your assets
|
|
60
|
-
config.assets.version = '1.0'
|
|
61
|
-
end
|
|
62
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# SQLite version 3.x
|
|
2
|
-
# gem install sqlite3
|
|
3
|
-
#
|
|
4
|
-
# Ensure the SQLite 3 gem is defined in your Gemfile
|
|
5
|
-
# gem 'sqlite3'
|
|
6
|
-
development:
|
|
7
|
-
adapter: sqlite3
|
|
8
|
-
database: db/development.sqlite3
|
|
9
|
-
pool: 5
|
|
10
|
-
timeout: 5000
|
|
11
|
-
|
|
12
|
-
# Warning: The database defined as "test" will be erased and
|
|
13
|
-
# re-generated from your development database when you run "rake".
|
|
14
|
-
# Do not set this db to the same as development or production.
|
|
15
|
-
test:
|
|
16
|
-
adapter: sqlite3
|
|
17
|
-
database: db/test.sqlite3
|
|
18
|
-
pool: 5
|
|
19
|
-
timeout: 5000
|
|
20
|
-
|
|
21
|
-
production:
|
|
22
|
-
adapter: sqlite3
|
|
23
|
-
database: db/production.sqlite3
|
|
24
|
-
pool: 5
|
|
25
|
-
timeout: 5000
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
TestApp::Application.configure do
|
|
2
|
-
# Settings specified here will take precedence over those in config/application.rb
|
|
3
|
-
|
|
4
|
-
# In the development environment your application's code is reloaded on
|
|
5
|
-
# every request. This slows down response time but is perfect for development
|
|
6
|
-
# since you don't have to restart the web server when you make code changes.
|
|
7
|
-
config.cache_classes = false
|
|
8
|
-
|
|
9
|
-
# Log error messages when you accidentally call methods on nil.
|
|
10
|
-
config.whiny_nils = true
|
|
11
|
-
|
|
12
|
-
# Show full error reports and disable caching
|
|
13
|
-
config.consider_all_requests_local = true
|
|
14
|
-
config.action_controller.perform_caching = false
|
|
15
|
-
|
|
16
|
-
# Don't care if the mailer can't send
|
|
17
|
-
config.action_mailer.raise_delivery_errors = false
|
|
18
|
-
|
|
19
|
-
# Print deprecation notices to the Rails logger
|
|
20
|
-
config.active_support.deprecation = :log
|
|
21
|
-
|
|
22
|
-
# Only use best-standards-support built into browsers
|
|
23
|
-
config.action_dispatch.best_standards_support = :builtin
|
|
24
|
-
|
|
25
|
-
# Raise exception on mass assignment protection for Active Record models
|
|
26
|
-
config.active_record.mass_assignment_sanitizer = :strict
|
|
27
|
-
|
|
28
|
-
# Log the query plan for queries taking more than this (works
|
|
29
|
-
# with SQLite, MySQL, and PostgreSQL)
|
|
30
|
-
config.active_record.auto_explain_threshold_in_seconds = 0.5
|
|
31
|
-
|
|
32
|
-
# Do not compress assets
|
|
33
|
-
config.assets.compress = false
|
|
34
|
-
|
|
35
|
-
# Expands the lines which load the assets
|
|
36
|
-
config.assets.debug = true
|
|
37
|
-
end
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
TestApp::Application.configure do
|
|
2
|
-
# Settings specified here will take precedence over those in config/application.rb
|
|
3
|
-
|
|
4
|
-
# Code is not reloaded between requests
|
|
5
|
-
config.cache_classes = true
|
|
6
|
-
|
|
7
|
-
# Full error reports are disabled and caching is turned on
|
|
8
|
-
config.consider_all_requests_local = false
|
|
9
|
-
config.action_controller.perform_caching = true
|
|
10
|
-
|
|
11
|
-
# Disable Rails's static asset server (Apache or nginx will already do this)
|
|
12
|
-
config.serve_static_assets = false
|
|
13
|
-
|
|
14
|
-
# Compress JavaScripts and CSS
|
|
15
|
-
config.assets.compress = true
|
|
16
|
-
|
|
17
|
-
# Don't fallback to assets pipeline if a precompiled asset is missed
|
|
18
|
-
config.assets.compile = false
|
|
19
|
-
|
|
20
|
-
# Generate digests for assets URLs
|
|
21
|
-
config.assets.digest = true
|
|
22
|
-
|
|
23
|
-
# Defaults to nil and saved in location specified by config.assets.prefix
|
|
24
|
-
# config.assets.manifest = YOUR_PATH
|
|
25
|
-
|
|
26
|
-
# Specifies the header that your server uses for sending files
|
|
27
|
-
# config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
|
|
28
|
-
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
|
|
29
|
-
|
|
30
|
-
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
|
|
31
|
-
# config.force_ssl = true
|
|
32
|
-
|
|
33
|
-
# See everything in the log (default is :info)
|
|
34
|
-
# config.log_level = :debug
|
|
35
|
-
|
|
36
|
-
# Prepend all log lines with the following tags
|
|
37
|
-
# config.log_tags = [ :subdomain, :uuid ]
|
|
38
|
-
|
|
39
|
-
# Use a different logger for distributed setups
|
|
40
|
-
# config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
|
|
41
|
-
|
|
42
|
-
# Use a different cache store in production
|
|
43
|
-
# config.cache_store = :mem_cache_store
|
|
44
|
-
|
|
45
|
-
# Enable serving of images, stylesheets, and JavaScripts from an asset server
|
|
46
|
-
# config.action_controller.asset_host = "http://assets.example.com"
|
|
47
|
-
|
|
48
|
-
# Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added)
|
|
49
|
-
# config.assets.precompile += %w( search.js )
|
|
50
|
-
|
|
51
|
-
# Disable delivery errors, bad email addresses will be ignored
|
|
52
|
-
# config.action_mailer.raise_delivery_errors = false
|
|
53
|
-
|
|
54
|
-
# Enable threaded mode
|
|
55
|
-
# config.threadsafe!
|
|
56
|
-
|
|
57
|
-
# Enable locale fallbacks for I18n (makes lookups for any locale fall back to
|
|
58
|
-
# the I18n.default_locale when a translation can not be found)
|
|
59
|
-
config.i18n.fallbacks = true
|
|
60
|
-
|
|
61
|
-
# Send deprecation notices to registered listeners
|
|
62
|
-
config.active_support.deprecation = :notify
|
|
63
|
-
|
|
64
|
-
# Log the query plan for queries taking more than this (works
|
|
65
|
-
# with SQLite, MySQL, and PostgreSQL)
|
|
66
|
-
# config.active_record.auto_explain_threshold_in_seconds = 0.5
|
|
67
|
-
end
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
TestApp::Application.configure do
|
|
2
|
-
# Settings specified here will take precedence over those in config/application.rb
|
|
3
|
-
|
|
4
|
-
# The test environment is used exclusively to run your application's
|
|
5
|
-
# test suite. You never need to work with it otherwise. Remember that
|
|
6
|
-
# your test database is "scratch space" for the test suite and is wiped
|
|
7
|
-
# and recreated between test runs. Don't rely on the data there!
|
|
8
|
-
config.cache_classes = true
|
|
9
|
-
|
|
10
|
-
# Configure static asset server for tests with Cache-Control for performance
|
|
11
|
-
config.serve_static_assets = true
|
|
12
|
-
config.static_cache_control = "public, max-age=3600"
|
|
13
|
-
|
|
14
|
-
# Log error messages when you accidentally call methods on nil
|
|
15
|
-
config.whiny_nils = true
|
|
16
|
-
|
|
17
|
-
# Show full error reports and disable caching
|
|
18
|
-
config.consider_all_requests_local = true
|
|
19
|
-
config.action_controller.perform_caching = false
|
|
20
|
-
|
|
21
|
-
# Raise exceptions instead of rendering exception templates
|
|
22
|
-
config.action_dispatch.show_exceptions = false
|
|
23
|
-
|
|
24
|
-
# Disable request forgery protection in test environment
|
|
25
|
-
config.action_controller.allow_forgery_protection = false
|
|
26
|
-
|
|
27
|
-
# Tell Action Mailer not to deliver emails to the real world.
|
|
28
|
-
# The :test delivery method accumulates sent emails in the
|
|
29
|
-
# ActionMailer::Base.deliveries array.
|
|
30
|
-
config.action_mailer.delivery_method = :test
|
|
31
|
-
|
|
32
|
-
# Raise exception on mass assignment protection for Active Record models
|
|
33
|
-
config.active_record.mass_assignment_sanitizer = :strict
|
|
34
|
-
|
|
35
|
-
# Print deprecation notices to the stderr
|
|
36
|
-
config.active_support.deprecation = :stderr
|
|
37
|
-
end
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
# Be sure to restart your server when you modify this file.
|
|
2
|
-
|
|
3
|
-
# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
|
|
4
|
-
# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
|
|
5
|
-
|
|
6
|
-
# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
|
|
7
|
-
# Rails.backtrace_cleaner.remove_silencers!
|