human_power 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d42507a07817c999567723432bcfc334acf0915
-  data.tar.gz: 16944ca90ecea9a3aa797787d03f321ed8cf49ef
+  metadata.gz: e0c1c83f98908134117b2b4aab263d7624fbdd8c
+  data.tar.gz: 622632e2dd563dacc9c1da16c45bc7c39063f7d2
 SHA512:
-  metadata.gz: 6687a32753b3261d45b899f02f14c3f7ba9a5195467f05281ec9e4acbdc43dde51bfe375f562bfac62800a7e8517a89f29b7c9d0643cdf58d7ebb6d6a98bb345
-  data.tar.gz: 836e6e7500a71d531eaf93c255f22074537b9a442752b51e4742b7987862ab45e958c9061935e8c1c80a52e5dc1cfdd01dfa2a06891f537a1bf466f3044412b1
+  metadata.gz: 17b7c4f6212ec8cd7c7b163cc4852490ae979a1c9c85f0186bf4fedcfba7b8846dfc442245c1bf89568101638a7e93c0d4d9921dc02b3544a25684842bfabec4
+  data.tar.gz: f0cec68d61d6e3fb520816e3add83874c6a61ed1c289ef3fefe342a60597b1e2d5c4437b7ac51aaf750330c2ef6ea6442f91c6c574804ca7e5fb6765c9d33b3c

data/README.md

@@ -77,6 +77,31 @@ Then visit `/robots.txt` in your browser.
 Please see [user_agents.yml](https://github.com/lassebunk/human_power/blob/master/user_agents.yml) for a list of 170+ built-in user agents/crawlers you can use like shown above.
 The list is from [UserAgentString.com](http://www.useragentstring.com/pages/Crawlerlist/).
 
+### Bot detection
+
+You can use the `HumanPower.is_bot?` method to check if a user agent is a known bot / crawler:
+
+```ruby
+# Googlebot
+ua = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+HumanPower.is_bot?(ua) # => true
+
+# Chrome
+ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36"
+HumanPower.is_bot?(ua) # => false
+
+# in Rails
+HumanPower.is_bot?(request.user_agent) # => performs check on current user agent
+```
+
+### Regular expression
+
+If you need to get a regular expression for bot detection, you can use:
+
+```ruby
+HumanPower.bot_regex # => regular expression that matches all known bots / crawlers
+```
+
 ## Caveats
 
 Human Power is great for adding rules to your robots.txt.

data/lib/human_power.rb

@@ -24,6 +24,19 @@ module HumanPower
       @user_agents ||= load_user_agents
     end
 
+    # Regular expression to match bot user agents.
+    def bot_regex
+      @bot_regex ||= begin
+        escaped_values = user_agents.values.map { |ua| Regexp.escape(ua) }
+        /#{escaped_values.join("|")}/i
+      end
+    end
+
+    # Returns +true+ if a given user agent is a bot.
+    def is_bot?(user_agent)
+      !!(user_agent =~ bot_regex)
+    end
+
   private
 
       # Loads the built-in user agents from crawlers.yml.

data/lib/human_power/version.rb

@@ -1,3 +1,3 @@
 module HumanPower
-  VERSION = "0.0.6"
+  VERSION = "0.1.0"
 end

data/test/bot_detection_test.rb

@@ -0,0 +1,43 @@
+require 'test_helper'
+
+class GeneratorTest < ActionView::TestCase
+  test "bot regex matches" do
+    bot_user_agents.each do |ua|
+      assert_match HumanPower.bot_regex, ua
+    end
+  end
+
+  test "bot regex browser matches" do
+    browser_user_agents.each do |ua|
+      assert_no_match HumanPower.bot_regex, ua
+    end
+  end
+
+  test "bot detection" do
+    bot_user_agents.each do |ua|
+      assert HumanPower.is_bot?(ua)
+    end
+  end
+
+  test "browser detection" do
+    browser_user_agents.each do |ua|
+      assert !HumanPower.is_bot?(ua)
+    end
+  end
+
+  private
+
+    def bot_user_agents
+      ["Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+       "Googlebot/2.1 (+http://www.googlebot.com/bot.html)",
+       "Mozilla/4.0 (compatible; B-l-i-t-z-B-O-T)",
+       "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"]
+    end
+
+    def browser_user_agents
+      ["Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
+       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
+       "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
+       "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0"]
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: human_power
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Lasse Bunk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-14 00:00:00.000000000 Z
+date: 2014-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -91,6 +91,7 @@ files:
 - lib/human_power/rails/engine.rb
 - lib/human_power/rule.rb
 - lib/human_power/version.rb
+- test/bot_detection_test.rb
 - test/dummy/README.rdoc
 - test/dummy/Rakefile
 - test/dummy/app/assets/images/.keep
@@ -159,6 +160,7 @@ signing_key:
 specification_version: 4
 summary: Easy generation of robots.txt. Force the robots into submission!
 test_files:
+- test/bot_detection_test.rb
 - test/dummy/README.rdoc
 - test/dummy/Rakefile
 - test/dummy/app/assets/images/.keep