robot_rules 0.9.1 → 0.9.2

data/Rakefile

@@ -9,6 +9,7 @@ begin
     gem.email = "edgargonzalez@gmail.com"
     gem.homepage = "http://github.com/hasmanydevelopers/robot_rules"
     gem.authors = ["James Edward Gray II", "Jeremy Friesen", "Edgar Gonzalez"]
+    gem.add_dependency('addressable', '>= 2.1.2')
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError

data/VERSION.yml

@@ -1,4 +1,5 @@
 --- 
 :major: 0
 :minor: 9
-:patch: 1
+:build: 
+:patch: 2

data/lib/robot_rules.rb

@@ -5,18 +5,17 @@
 #  Created by James Edward Gray II on 2006-01-31.
 #  Copyright 2006 Gray Productions. All rights reserved.
 
-require "uri"
+require 'addressable/uri'
 
 # Based on Perl's WWW::RobotRules module, by Gisle Aas.
 class RobotRules
    def initialize( user_agent )
-     @user_agent = user_agent.scan(/\S+/).first.sub(%r{/.*},  
-"").downcase
+     @user_agent = user_agent.scan(/\S+/).first.sub(%r{/.*},"").downcase
      @rules      = Hash.new { |rules, rule| rules[rule] = Array.new }
    end
 
-   def parse( text_uri, robots_data )
-     uri      = URI.parse(text_uri)
+   def parse( site, robots_data )
+     uri      = site.kind_of?(Addressable::URI) ? site : Addressable::URI::parse(site)
      location = "#{uri.host}:#{uri.port}"
      @rules.delete(location)
 
@@ -64,8 +63,8 @@ class RobotRules
      end
    end
 
-   def allowed?( text_uri )
-     uri      = URI.parse(text_uri)
+   def allowed?( u )
+     uri      = u.kind_of?(Addressable::URI) ? u : Addressable::URI::parse(u)
      location = "#{uri.host}:#{uri.port}"
      path     = uri.path
 
@@ -74,3 +73,4 @@ class RobotRules
      not @rules[location].any? { |rule| path.index(rule) == 0 }
    end
 end
+

data/robot_rules.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{robot_rules}
-  s.version = "0.9.1"
+  s.version = "0.9.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Edward Gray II", "Jeremy Friesen", "Edgar Gonzalez"]
-  s.date = %q{2010-07-09}
+  s.date = %q{2010-07-21}
   s.email = %q{edgargonzalez@gmail.com}
   s.extra_rdoc_files = [
     "LICENSE",
@@ -42,9 +42,12 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<addressable>, [">= 2.1.2"])
     else
+      s.add_dependency(%q<addressable>, [">= 2.1.2"])
     end
   else
+    s.add_dependency(%q<addressable>, [">= 2.1.2"])
   end
 end
 

data/test/robot_rules_test.rb

@@ -23,7 +23,7 @@ class RobotRulesTest < Test::Unit::TestCase
     @robot_rule = RobotRules.new('Microsoft')
     robots_txt = %(/robots.txt:\nUser-agent: Microsoft\nDisallow:  google\nUser-agent:  *\nDisallow:    images)
     @robot_rule.parse(File.join(SITE_URL,'robots.txt'), robots_txt)
-    
+
     assert_equal false, @robot_rule.allowed?(File.join(SITE_URL, 'google/hellow_world.txt'))
   end
 
@@ -31,7 +31,27 @@ class RobotRulesTest < Test::Unit::TestCase
     @robot_rule = RobotRules.new('Google')
     robots_txt = %(/robots.txt:\nUser-agent: Microsoft\nDisallow:  google\nUser-agent:  *\nDisallow:    images)
     @robot_rule.parse(File.join(SITE_URL,'robots.txt'), robots_txt)
-    
+
     assert_equal true, @robot_rule.allowed?(File.join(SITE_URL, 'google/hellow_world.txt'))
   end
+
+  def test_should_allow_user_agent_when_disallowed_all_for_specific_user_agent
+    @robot_rule = RobotRules.new('MyBot')
+    robots_txt = "User-agent: mxbot\nDisallow: /"
+    @robot_rule.parse("#{SITE_URL}/robots.txt", robots_txt)
+
+    assert_equal true, @robot_rule.allowed?("#{SITE_URL}/hellow_world")
+  end
+
+  def test_using_uri_as_parameters_should_allow_user_agent_to_specified_path
+    robots_url = Addressable::URI::parse("#{SITE_URL}/robots.txt")
+    url = Addressable::URI::parse("#{SITE_URL}/google/hellow_world.txt")
+    @robot_rule = RobotRules.new('Google')
+    robots_txt = %(/robots.txt:\nUser-agent: Microsoft\nDisallow:  google\nUser-agent:  *\nDisallow:    images)
+    @robot_rule.parse(robots_url, robots_txt)
+
+    assert_equal true, @robot_rule.allowed?(url)
+  end
+
 end
+

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 1
-  version: 0.9.1
+  - 2
+  version: 0.9.2
 platform: ruby
 authors: 
 - James Edward Gray II
@@ -16,10 +16,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-09 00:00:00 -04:30
+date: 2010-07-21 00:00:00 -04:30
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  type: :runtime
+  version_requirements: *id001
 description: 
 email: edgargonzalez@gmail.com
 executables: []