is_crawler 0.1.2 → 0.1.3

data/README.md

@@ -1,7 +1,6 @@
-# IsCrawler
-[![Gem Version](https://badge.fury.io/rb/is_crawler.png)](http://badge.fury.io/rb/is_crawler) [![Code Climate](https://codeclimate.com/github/ccashwell/is_crawler.png)](https://codeclimate.com/github/ccashwell/is_crawler) [![Build Status](https://travis-ci.org/ccashwell/is_crawler.png?branch=master)](https://travis-ci.org/ccashwell/is_crawler)
+# is_crawler [![Gem Version](https://badge.fury.io/rb/is_crawler.png)](http://badge.fury.io/rb/is_crawler) [![Build Status](https://travis-ci.org/ccashwell/is_crawler.png?branch=master)](https://travis-ci.org/ccashwell/is_crawler) [![Code Climate](https://codeclimate.com/github/ccashwell/is_crawler.png)](https://codeclimate.com/github/ccashwell/is_crawler)
 
-is\_crawler does exactly what you might think it does: determine if the supplied string matches a known crawler or bot.
+This gem does one thing: determine if the supplied string matches a known crawler or bot. It matches against a very short list of strings found in the user agents that represent over 95% of crawler traffic. IMO, if it ain't detected, it ain't important.
 
 ## Installation
 
@@ -24,7 +23,7 @@ You can use the `is_crawler?` method with just a user agent string to determine
     class MyController < ActionController::Base
       include IsCrawler
       def index
-        if is_any_crawler? request.env["HTTP_USER_AGENT"]
+        if is_crawler? request.env["HTTP_USER_AGENT"]
           render 'special_crawler_index'
         else
           render 'normal_boring_index'
@@ -38,7 +37,7 @@ You can use the `is_crawler?` method with just a user agent string to determine
 
 You can also define custom crawlers like this:
 
-    Crawler::CUSTOM << Crawler.new(:custom_crawler_name, "string that is always present in crawler requests")
+    Crawler::CUSTOM << Crawler.new(:custom_crawler_name, "string that is always present in the crawler's user agent")
 
 That's it!
 

data/lib/config/crawlers.yml

@@ -1,34 +1,22 @@
-addthis:
-  ua_string: AddThis.com
-alexa:
-  ua_string: ia_archiver
-archive_org:
-  ua_string: archive.org_bot
-bing:
-  ua_string: bingbot
-bitly:
-  ua_string: bitlybot
-exabot:
-  ua_string: Exabot
-facebook:
-  ua_string: facebookexternalhit
-flipboard:
-  ua_string: FlipboardProxy
-google:
-  ua_string: Googlebot
-google_web_preview:
-  ua_string: Google Web Preview
-msn:
-  ua_string: MSNBot
-openwebspider:
-  ua_string: OpenWebSpider
-technorati:
-  ua_string: Technoratibot
-twitter:
-  ua_string: Twitterbot
-yahoo:
-  ua_string: Yahoo! Slurp
-yahoo_jp:
-  ua_string: Y!J
-yandex:
-  ua_string: Yandex
+crawlers:
+  addthis: AddThis.com
+  alexa: ia_archiver
+  archive_org: archive.org_bot
+  ask: Ask Jeeves
+  baidu: baidu
+  bing: bingbot
+  bitly: bitlybot
+  blekko: Blekkobot
+  exabot: Exabot
+  facebook: facebookexternalhit
+  flipboard: FlipboardProxy
+  google: Googlebot
+  google_web_preview: Google Web Preview
+  msn: MSNBot
+  mywebsearch: MyWebSearch
+  openwebspider: OpenWebSpider
+  technorati: Technoratibot
+  twitter: Twitterbot
+  yahoo: Yahoo! Slurp
+  yahoo_jp: Y!J
+  yandex: Yandex

data/lib/crawler.rb

@@ -1,8 +1,8 @@
 require 'yaml'
 
 class Crawler < Struct.new(:name, :ua_string)
-  DEFAULT = YAML.load(File.read(File.expand_path('../config/crawlers.yml', __FILE__))).collect do |k,v|
-    Crawler.new(k.to_sym, v["ua_string"])
+  DEFAULT = YAML.load(File.read(File.expand_path('../config/crawlers.yml', __FILE__)))["crawlers"].collect do |k,v|
+    Crawler.new(k.to_sym, v)
   end
 
   CUSTOM = []
@@ -17,7 +17,7 @@ class Crawler < Struct.new(:name, :ua_string)
     end
 
     def which_crawler user_agent
-      all.detect {|crawler| crawler.matches? user_agent }.name rescue nil
+      all.detect { |crawler| crawler.matches? user_agent }.name rescue nil
     end
   end
 

data/lib/is_crawler/version.rb

@@ -1,3 +1,3 @@
 module IsCrawler
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/lib/is_crawler.rb

@@ -6,7 +6,7 @@ module IsCrawler
     if specific_crawlers && specific_crawlers.size > 0
       specific_crawlers.include?(Crawler.which_crawler(requesting_user_agent))
     else
-      Crawler.matches_any?(requesting_user_agent) unless specific_crawlers.size > 0
+      Crawler.matches_any?(requesting_user_agent)
     end
   end
 end

data/spec/lib/crawler_spec.rb

@@ -1,62 +1,69 @@
 describe Crawler do
-  let(:user_agent) { "Commodo Vestibulum/1.0" }
+  let(:chrome_user_agent) { 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17' }
+  let(:google_user_agent) { 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }
+  let(:facebook_user_agent) { 'facebookexternalhit/1.0 (http://www.facebook.com/externalhit_uatext.php)' }
 
   describe '.matches_any?' do
     subject { Crawler.matches_any?(user_agent) }
 
     context 'When an unknown user agent is encountered' do
+      let(:user_agent) { chrome_user_agent }
       it { should be_false }
     end
 
     context 'When a known user agent is encountered' do
-      Crawler.all.each do |crawler|
-        context "such as the #{crawler.name.to_s} bot" do
-          let(:user_agent) { "#{crawler.ua_string}" }
-          it { should be_true }
-        end
+      context 'such as the facebook crawler' do
+        let(:user_agent) { facebook_user_agent }
+        it { should be_true }
+      end
+
+      context 'such as the Googlebot' do
+        let(:user_agent) { google_user_agent }
+        it { should be_true }
       end
     end
   end
 
   describe '#matches?' do
-    Crawler.all.each do |crawler|
-      describe "Comparing #{crawler.name.to_s}'s known UA string" do
-        subject { crawler.matches?(user_agent) }
-        context "with a string containing '#{crawler.ua_string}'" do
-          let(:user_agent) { "Mozilla/5.0 #{crawler.ua_string}/1.1 (KHTML, like Gecko)" }
-          it { should be_true }
-        end
+    describe 'Comparing Googlebot\'s known UA string' do
+      subject { Crawler.new(:google, 'Googlebot').matches?(user_agent) }
+      context "with a matching string" do
+        let(:user_agent) { google_user_agent }
+        it { should be_true }
+      end
 
-        context 'with a non-matching string' do
-          it { should be_false }
-        end
+      context 'with a non-matching string' do
+        let(:user_agent) { chrome_user_agent }
+        it { should be_false }
       end
     end
   end
 
-  describe '#which_crawler' do
+  describe '.which_crawler' do
     subject { Crawler.which_crawler(user_agent) }
     context 'When the provided string matches a crawler' do
-      let(:user_agent) { "facebookexternalhit/1.1" }
+      let(:user_agent) { facebook_user_agent }
       it { should be :facebook }
     end
 
     context 'When the provided string matches no crawlers' do
+      let(:user_agent) { chrome_user_agent }
       it { should be_nil }
     end
   end
 
   describe 'Custom Crawler' do
     let(:custom_crawler) { Crawler.new(:custom, "Custom/1.0") }
+    let(:user_agent) { custom_crawler.ua_string }
     before { Crawler::CUSTOM << custom_crawler }
     context '.matches_any' do
       subject { Crawler.matches_any?(user_agent) }
       context 'When the provided string matches the custom crawler' do
-        let(:user_agent) { "Custom/1.0" }
         it { should be_true }
       end
 
       context 'When the provided string does not match the custom crawler' do
+        let(:user_agent) { chrome_user_agent }
         it { should be_false }
       end
     end
@@ -64,11 +71,11 @@ describe Crawler do
     context '.which_crawler' do
       subject { Crawler.which_crawler(user_agent) }
       context 'When the provided string matches the custom crawler' do
-        let(:user_agent) { "Custom/1.0" }
         it { should be custom_crawler.name }
       end
 
       context 'When the provided string does not match the custom crawler' do
+        let(:user_agent) { chrome_user_agent }
         it { should_not be custom_crawler.name }
       end
     end
@@ -76,11 +83,11 @@ describe Crawler do
     context '#matches?' do
       subject { custom_crawler.matches?(user_agent) }
       context 'When the provided string matches the custom crawler' do
-        let(:user_agent) { "Custom/1.0" }
         it { should be_true }
       end
 
       context 'When the provided string does not match the custom crawler' do
+        let(:user_agent) { chrome_user_agent }
         it { should be_false }
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: is_crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-03 00:00:00.000000000 Z
+date: 2013-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -77,7 +77,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3983243957140806942
+      hash: -1826155163722186205
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -86,7 +86,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3983243957140806942
+      hash: -1826155163722186205
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24