browser 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8da651ebe15cf30e9eccffd7e45c2b2659785776
-  data.tar.gz: 08e9184e851b07fff6b2ff4efda0a820e042df50
+  metadata.gz: a8f4650e2d1a3d6e81011637c57feea62b92444b
+  data.tar.gz: 6383b04357145bfd6acf774c235b1af3b71d4399
 SHA512:
-  metadata.gz: 1af5f14bd4c40f26773c2aecebf58802e3cc4f037481bfec8240c5cb54c52673a23249909d7280d13a1a7a0094942f2384ccf4e1e8a835c703bf6b9d5149dd22
-  data.tar.gz: b910d7e7b92cbc5456747369c27a1f955a1d33329b61caa2f50590aca90f20c80f13d75f6f8bab56b5e595b5ebaebe3437639f7dc8df0870090f33e67daad50b
+  metadata.gz: 69effb2ac78f0bd154b38aa43a479dc5f42e29fce57bf702fe8521981122cca202e4fa35692a3f6e1b2e359906387cc086ee9b4a8102d2d0cf67819fdf9a718e
+  data.tar.gz: b2c1172bc85607ceb885006a842837ac68d2e54c5443ab9885bcc17cd2c77a3aa779836d57f4037f9f72a1c561add97ce3221c25843af5362ed6f498de48e374

data/README.md

@@ -18,7 +18,7 @@ require "rubygems"
 require "browser"
 
 browser = Browser.new(:ua => "some string", :accept_language => "en-us")
-browser.name        # readable browser name
+browser.name            # readable browser name
 browser.version
 browser.safari?
 browser.opera?
@@ -29,20 +29,21 @@ browser.tablet?
 browser.console?
 browser.firefox?
 browser.ie?
-browser.ie6?        # this goes up to 11
-browser.modern?     # Webkit, Firefox 17+, IE 9+ and Opera 12+
-browser.platform    # return :mac, :windows, :linux or :other
+browser.ie6?            # this goes up to 11
+browser.modern?         # Webkit, Firefox 17+, IE 9+ and Opera 12+
+browser.platform        # return :mac, :windows, :linux or :other
 browser.mac?
 browser.windows?
 browser.windows_x64?
 browser.linux?
 browser.blackberry?
 browser.bot?
-browser.meta        # an array with several attributes
-browser.to_s        # the meta info joined by space
+browser.search_engine?
+browser.meta            # an array with several attributes
+browser.to_s            # the meta info joined by space
 ```
 
-See the [tests](https://github.com/fnando/browser/blob/master/test/browser_test.rb) and [implementation](https://github.com/fnando/browser/blob/master/lib/browser.rb) for more examples.
+See the [tests](https://github.com/fnando/browser/blob/master/test/browser_spec.rb) and [implementation](https://github.com/fnando/browser/blob/master/lib/browser.rb) for more examples.
 
 ### What defines a modern browser?
 
@@ -78,6 +79,14 @@ This adds a helper method called `browser`, that inspects your current user agen
 <% end %>
 ```
 
+### Bots
+
+Browser used to detect empty user agents as bots, but this behaviour has changed. If you want to bring this detection back, you can activate it through the following call:
+
+```ruby
+Browser::Bots.detect_empty_ua!
+```
+
 ### Middleware
 
 You can use the `Browser::Middleware` to redirect user agents.
@@ -100,6 +109,7 @@ Notice that you can have multiple conditionals.
 
 ```ruby
 Rails.configuration.middleware.use Browser::Middleware do
+  next if browser.search_engine?
   redirect_to upgrade_path(browser: "oldie") if browser.ie? && !browser.modern?
   redirect_to upgrade_path(browser: "oldfx") if browser.firefox? && !browser.modern?
 end

data/bots.yml

@@ -11,20 +11,23 @@ Baiduspider: "Chinese search engine"
 bingbot: "Microsoft bing bot"
 bitlybot: "bit.ly bot"
 bot@linkfluence.net: "Linkfluence bot"
+BuiBui-CheckBot: "email: buibui[at]dadapro[dot]com"
 Butterfly: "Topsy Labs"
 buzztalk: "buzztalk"
 check_http: "Nagios monitor"
 CMRadar/0.1: "CMRadar/0.1"
 ColdFusion: "ColdFusion http library"
-Crowsnest: "crowsnest.tv news portal bot"
 Crowsnest: "Crowsnest"
+Crowsnest: "crowsnest.tv news portal bot"
 curl: "curl unix CLI http client"
 developers.google.com/+/web/snippet/: "Google Plus"
 DigitalPersona Fingerprint Software: "HP Fingerprint scanner"
+DotBot: "Dot Bot"
 Embedly: "Embedly"
 EOAAgent: "EOAAgent"
 EventMachine HttpClient: "Ruby http library"
 EvriNid: "Evri bot"
+Exabot: "Exalead's bot"
 ExaleadCloudView: "ExaleadCloudView"
 facebookexternalhit: "facebook bot"
 FeedBurner: "RSS bot"
@@ -50,29 +53,33 @@ Insieve Bot: "Insieve Bot"
 InsieveBot: "InsieveBot"
 Instapaper: "Instapaper"
 jack: "jack"
-Jakarta Commons: "Jakarta Commons HttpClient"
 Jakarta Commons-HttpClient: "Jakarta Commons-HttpClient"
+Jakarta Commons: "Jakarta Commons HttpClient"
 Java: "Generic Java http library"
 Jetslide: "Jetslide"
 JS-Kit: "URL resolver"
-Kimengi: "nineconnections.com bot"
 Kimengi/nineconnections.com: "Kimengi/nineconnections.com"
+Kimengi: "nineconnections.com bot"
 knows.is: "knows.is"
 kraken: "kraken"
 Laconica: "Laconica"
 libwww-perl: "Perl client-server library loved by script kids"
+linkdexbot: "Linkdex Bot"
 LinkedInBot: "LinkedIn"
 LinksCrawler: "LinksCrawler"
-Livelapbot: "Livelapbot"
 Lipperhey: "Lipperhey"
+Livelapbot: "Livelapbot"
+LoadTimeBot: "Load Time Bot"
 LongURL: "URL expander service"
 Lumibot: "Lumibot"
 lwp-trivial: "Another Perl library loved by script kids"
+Mail.RU_Bot: "Mail.ru Bot"
 Mediapartners-Google: "Google Adsense bot"
-MetaURI: "URI meta information extractor bot"
 MetaURI: "MetaURI"
+MetaURI: "URI meta information extractor bot"
 MFE_expand: "Mcafee spider"
 MJ12bot: "Majestic-12 spider"
+MojeekBot: "Mojeek UK search crawler"
 msnbot: "Microsoft bot"
 Netvibes: "Personalized dashboard bot"
 NewRelicPinger: "NewRelic monitor"
@@ -96,9 +103,11 @@ QuickLook: "QuickLook"
 Readability: "Readability"
 RebelMouse: "RebelMouse"
 redditbot: "Reddit Bot"
+Riddler: "Riddler Bot"
 rogerbot: "SeoMoz spider"
-Ruby: "Ruby"
 Ruby/1.9.3: "Ruby/1.9.3"
+Ruby: "Ruby"
+Scrapy: "Scrapy"
 SemrushBot: "SEO analysis bot"
 ShortLinkTranslate: "Link shortener"
 ShowyouBot: "Showyou iOS app spider"
@@ -107,6 +116,7 @@ SiteUptime: "Site monitoring services"
 Slurp: "Yahoo spider"
 Sogou: "Chinese search engine"
 spider: "generic web spider"
+Spinn3r: "Spinn3r aggregator"
 Test Certificate Info: "C http library?"
 trendictionbot: "trendiction search"
 TweetmemeBot: "TweetMeMe Crawler"
@@ -123,7 +133,10 @@ vkShare: "VKontake Sharer"
 Webscout: "Webscout"
 Wget: "wget unix CLI http client"
 WordPress: "WordPress spider"
+Xenu Link Sleuth: "Xenu Link Sleuth"
 YandexBot: "Yandex spider"
+YandexDirect: "Yandex Direct spider"
+YandexMetrika: "Yandex Metrika spider"
 YOURLS: "YOURLS"
 zelist.ro: "feed parser"
 ZIBB: "ZIBB spider"

data/lib/browser.rb

@@ -1,5 +1,6 @@
 require "set"
 require "yaml"
+require "pathname"
 
 require "browser/middleware"
 require "browser/middleware/context"

data/lib/browser/methods/bots.rb

@@ -1,9 +1,19 @@
 class Browser
   module Bots
-    BOTS = YAML.load_file(File.expand_path("../../../../bots.yml", __FILE__))
+    root = Pathname.new(File.expand_path("../../../..", __FILE__))
+    BOTS = YAML.load_file(root.join("bots.yml"))
+    SEARCH_ENGINES = YAML.load_file(root.join("search_engines.yml"))
+
+    def self.detect_empty_ua!
+      Browser::Bots::BOTS[''] = 'Generic bot'
+    end
 
     def bot?
-      ua.empty? || BOTS.any? {|key, _| ua.include?(key) }
+      BOTS.any? {|key, _| ua.include?(key) }
+    end
+
+    def search_engine?
+      SEARCH_ENGINES.any? {|key, _| ua.include?(key) }
     end
   end
 end

data/lib/browser/version.rb

@@ -1,7 +1,7 @@
 class Browser
   module Version
     MAJOR = 0
-    MINOR = 5
+    MINOR = 6
     PATCH = 0
     STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
   end

data/search_engines.yml

@@ -0,0 +1,6 @@
+Baidu: "Chinese search engine"
+bingbot: "Microsoft bing bot"
+Googlebot: "Google spider"
+Slurp: "Yahoo spider"
+DuckDuckBot: "Duck Duck Go"
+Ask Jeeves: "Ask Jeeves"

data/test/browser_spec.rb

@@ -814,21 +814,36 @@ describe Browser do
   end
 
   it "detects bots" do
-    @browser.ua = $ua["GOOGLE_BOT"]
-    assert @browser.bot?
+    %w[
+      DOT_BOT
+      FACEBOOK_BOT
+      GOOGLE_BOT
+      LINKDEXBOT
+      LOAD_TIME_BOT
+      MAIL_RU
+      MSN_BOT
+      SCRAPY
+      YANDEX_DIRECT
+      YANDEX_METRIKA
+    ].each do |key|
+      @browser.ua = $ua[key]
+      assert @browser.bot?, "#{$ua[key]} should be a bot"
+    end
 
-    @browser.ua = $ua["MSN_BOT"]
-    assert @browser.bot?
+    @browser.ua = $ua["CHROME"]
+    assert ! @browser.bot?
+  end
 
-    @browser.ua = $ua["FACEBOOK_BOT"]
-    assert @browser.bot?
+  it "doesn't consider empty UA as bot" do
+    @browser.ua = ''
+    assert ! @browser.bot?
+  end
 
-    # Many bots actually report empty ua strings.
+  it "allows setting empty string as bots" do
+    Browser::Bots.detect_empty_ua!
     @browser.ua = ''
     assert @browser.bot?
-
-    @browser.ua = $ua["CHROME"]
-    assert ! @browser.bot?
+    Browser::Bots::BOTS.delete('')
   end
 
   it "detects chrome os" do
@@ -840,4 +855,18 @@ describe Browser do
     @browser.ua = $ua["SMART_TV"]
     assert @browser.tv?
   end
+
+  it "detects as search engines" do
+    %w[
+      ASK
+      BAIDU
+      BINGBOT
+      DUCKDUCKGO
+      GOOGLE_BOT
+      YAHOO_SLURP
+    ].each do |key|
+      @browser.ua = $ua[key]
+      assert @browser.search_engine?, "#{$ua[key]} should be a search engine"
+    end
+  end
 end

data/test/ua.yml

@@ -65,3 +65,15 @@ GOOGLE_BOT: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.
 MSN_BOT: "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)"
 FACEBOOK_BOT: "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
 SMART_TV: "Mozilla/5.0 (SmartHub; SMART-TV; U; Linux/SmartTV) AppleWebKit/531.2+ (KHTML, like Gecko) WebBrowser/1.0 SmartTV Safari/531.2+"
+DOT_BOT: "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"
+LINKDEXBOT: "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)"
+LOAD_TIME_BOT: "Mozilla/5.0 (compatible; LoadTimeBot/0.9; +http://www.loadtime.net/bot.html)"
+MAIL_RU: "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)"
+SCRAPY: "Scrapy/0.18.4 (+http://scrapy.org)"
+YANDEX_DIRECT: "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)"
+YANDEX_METRIKA: "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)"
+BAIDU: "Baiduspider+(+http://www.baidu.com/search/spider.htm)"
+YAHOO_SLURP: "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
+ASK: "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)"
+BINGBOT: "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
+DUCKDUCKGO: "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: browser
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Nando Vieira
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-20 00:00:00.000000000 Z
+date: 2014-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -124,6 +124,7 @@ files:
 - lib/browser/middleware/context/url_methods.rb
 - lib/browser/rails.rb
 - lib/browser/version.rb
+- search_engines.yml
 - test/browser_spec.rb
 - test/middleware_spec.rb
 - test/sample_app.rb