bot_detection 0.9.9 → 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d3eb0c816dab6abd7bedf902b7064e55d87b6b3
+  data.tar.gz: 31db977a5485d47cf579822b3e6a46297730ff91
+SHA512:
+  metadata.gz: 9dec5f7768f8a9d382c7a6faa045977e649b89e8e05a0f25f90224f97333e65a33edf023960162c91a11e21c0ebc78ec0f145cba22b06f20c8f71687e670f81a
+  data.tar.gz: ce183a9aa5f595fe46eaa1e9a3a7de8972b4a86d22cb66c4c29e3bf591e80f6f3b80790ad12808b8fb739d1a1d0d8732d1cc7d1815b6cd09477a9eeff0816f74

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+# Changelog
+
+#### Release 1.0.1
+
+- Yandex support added
+- Baidu support added
+- better check for hostnames on reverse lookup
+- match for full user agent on msn, bing and yahoo
+- it should now properly detect all google bots
+- the methods is_msn?, is_bing? and is_yahoo? have been removed
+
+#### Release 0.9.9
+
+- Initial version

data/README.md

@@ -16,6 +16,10 @@ Or install it yourself as:
 
     $ gem install bot_detection
 
+## Changes
+
+See the [CHANGELOG.md](CHANGELOG.md) file for details.
+
 ## Usage
 
 TODO: Write usage instructions here

data/bot_detection.gemspec

@@ -16,6 +16,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "public_suffix", "~> 1.4.6"
+
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
 end

data/lib/bot_detection/bot_user_agents.rb

@@ -0,0 +1,7 @@
+BotDetection::BotUserAgents = [
+  "FacebookExternalHit/1.1",
+  "FacebookExternalHit/1.0",
+  "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",
+  "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
+  "facebookplatform/1.0 (+http://developers.facebook.com)"
+]

data/lib/bot_detection/google_user_agents.rb

@@ -1,5 +1,6 @@
 BotDetection::GOOGLE_USER_AGENTS = [
   "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Googlebot/2.1 (+http://www.googlebot.com/bot.html)",
   "Googlebot/2.1 (+http://www.google.com/bot.html)",
   "Googlebot-News",
   "Googlebot-Image/1.0",

data/lib/bot_detection/instance_methods.rb

@@ -1,40 +1,39 @@
 require 'bot_detection'
 require_relative 'google_user_agents.rb'
+require_relative 'bot_user_agents.rb'
+require_relative 'search_engine_user_agents.rb'
 
 module BotDetection::InstanceMethods  
+  def is_known_crawler?
+    BotDetection::BotUserAgents.include?(user_agent) || is_search_engine_crawler?(reverse_lookup: false)
+  end
+  
   def is_search_engine_crawler? options = {}
     remote_ip = options.delete(:ip) || options.delete(:ip_address) || request.remote_ip
-    return false if remote_ip.blank? || (!is_google? && !is_yahoo? && !is_msn? && !is_bing?)
-
+    return false if remote_ip.blank?
+    
+    reverse_lookup = options.delete(:reverse_lookup)
+    reverse_lookup = true if reverse_lookup.nil?
+    
+    return false unless is_known_search_engine_crawler?    
+    return true unless reverse_lookup
     return true if options.delete(:development)
 
-    found = false
-    host = get_hostname(remote_ip)
-    ["crawl.yahoo.net", "googlebot", "search.msn.com", "ask.com"].each do |h|
-      found = true and break if host.include?(h)
-    end
-
-    return false unless found
+    host   = get_hostname(remote_ip)
+    domain = PublicSuffix.parse(host) rescue nil
+    return false if domain.nil?
     
-    host_ip = get_hostip(host)
-    return host_ip == remote_ip
-    false
+    return false unless ["crawl.yahoo.net", "googlebot.com", "google.com", "search.msn.com", "ask.com", "yandex.net", "yandex.com", "yandex.ru", "baidu.com", "baidu.jp"].include?(domain.domain.downcase)
+    get_hostip(host) == remote_ip
   end
 
   def is_google?
-    BotDetection::GOOGLE_USER_AGENTS.include?(request.env['HTTP_USER_AGENT']) || request.user_agent.to_s.downcase.include?("googlebot")
-  end
-  
-  def is_yahoo?
-    request.user_agent.to_s.downcase.include?("yahoo! slurp")
-  end
-  
-  def is_msn?
-    request.user_agent.to_s.downcase.include?("msnbot")
+    BotDetection::GOOGLE_USER_AGENTS.include?(user_agent)
   end
-  
-  def is_bing?
-    request.user_agent.to_s.downcase.include?("bing")
+
+protected
+  def is_known_search_engine_crawler?
+    is_google? || BotDetection::SearchEngineUserAgents.include?(user_agent)
   end
   
   def get_hostname(ip_address)
@@ -44,4 +43,8 @@ module BotDetection::InstanceMethods
   def get_hostip(host)
     Socket.gethostbyname(host).last.unpack("C*").join(".")    
   end
+
+  def user_agent
+    (request.env['HTTP_USER_AGENT'] || request.user_agent).to_s
+  end
 end

data/lib/bot_detection/search_engine_user_agents.rb

@@ -0,0 +1,36 @@
+BotDetection::SearchEngineUserAgents = [
+  "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
+  "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
+  "BaiDuSpider",
+  "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
+  "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexDirect/2.0; Dyatel; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexNews/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexCatalog/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexZakladki/3.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)",
+  "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+  "Mozilla/5.0 (compatible; Yahoo Slurp;   http://help.yahoo.com/help/us/ysearch/slurp)",
+  "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
+  "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+  "Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)",
+  "msnbot/2.1",
+  "msnbot/2.0b",
+  "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
+  "msnbot/1.1",
+  "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
+  "msnbot/0.9 (+http://search.msn.com/msnbot.htm)",
+  "msnbot/0.11 ( http://search.msn.com/msnbot.htm)",
+  "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)",
+]

data/lib/bot_detection/version.rb

@@ -1,3 +1,3 @@
 module BotDetection
-  VERSION = "0.9.9"
+  VERSION = "1.0.1"
 end

data/lib/bot_detection.rb

@@ -1,4 +1,5 @@
 require "bot_detection/version"
+require "public_suffix"
 
 module BotDetection
   def self.included(target)

metadata

@@ -1,20 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: bot_detection
 version: !ruby/object:Gem::Version
-  version: 0.9.9
-  prerelease: 
+  version: 1.0.1
 platform: ruby
 authors:
 - Nils Berenbold
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-07 00:00:00.000000000 Z
+date: 2014-11-13 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: public_suffix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +34,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,17 +41,15 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -51,38 +60,40 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bot_detection.gemspec
 - lib/bot_detection.rb
+- lib/bot_detection/bot_user_agents.rb
 - lib/bot_detection/google_user_agents.rb
 - lib/bot_detection/instance_methods.rb
+- lib/bot_detection/search_engine_user_agents.rb
 - lib/bot_detection/version.rb
 homepage: http://www.nilsberenbold.de
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.14
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Detects Search Engine crawlers by reverse DNS lookups.
 test_files: []