rdaneel 0.1.0 → 0.1.2

data/README.rdoc

@@ -1,6 +1,6 @@
 = R.Daneel
 
-An EventMachine+Ruby library to fetch urls following robots.txt rules.
+An EventMachine+Ruby library to fetch urls obeying robots.txt rules.
 
 RDaneel is built it on top of @igrigorik's {em-http-request}[http://github.com/igrigorik/em-http-request]
 

data/Rakefile

@@ -11,7 +11,7 @@ begin
     gem.homepage = "http://github.com/hasmanydevelopers/RDaneel"
     gem.authors = ["Edgar Gonzalez", "Anibal Rojas"]
     gem.add_dependency("em-http-request", ">= 0.2.10")
-    gem.add_dependency('robot_rules', '>= 0.9.1')
+    gem.add_dependency('robot_rules', '>= 0.9.3')
     gem.add_development_dependency "rspec", ">= 1.2.9"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.2

data/lib/rdaneel.rb

@@ -21,6 +21,7 @@ class RDaneel
 
   def initialize(uri)
     @uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri)
+    @uri.path = "/" if @uri.path.nil? || @uri.path == ""
     @redirects = []
   end
 
@@ -61,39 +62,53 @@ class RDaneel
         end
       else
         # other error
-        self.http_client = h
+        @http_client = h
         @error = "not success and not redirect"
         fail(self)
       end
     }
     _get = lambda {
-      if robots_cache && robots_file = robots_cache[robots_txt_url(current_uri)]
-        if robots_allowed?(robots_file, useragent, current_uri)
-          h = EM::HttpRequest.new(current_uri).get(options)
-          h.callback(&_handle_uri_callback)
-          h.errback {
-            @http_client = h
-            @error = h.error
+      robots_url = robots_txt_url(current_uri)
+      if robots_cache && robots_file = robots_cache[robots_url.to_s]
+        if robots_allowed?(robots_file, useragent, robots_url, current_uri)
+          begin
+            h = EM::HttpRequest.new(current_uri).get(options)
+            h.callback(&_handle_uri_callback)
+            h.errback {
+              @http_client = h
+              @error = h.error
+              fail(self)
+            }
+          rescue StandardError => se
+            @http_client = EM::HttpClient.new("")
+            @error = "#{se.message}\n#{se.backtrace.inspect}"
             fail(self)
-          }
+          end
         else
           @http_client = EM::HttpClient.new("")
           @error = "robots denied"
           fail(self)
         end
       else
-        robots = EM::HttpRequest.new(robots_txt_url(current_uri)).get
+        robots_url = robots_txt_url(current_uri)
+        robots = EM::HttpRequest.new(robots_url).get
         robots.callback {
           robots_file = robots.response
-          robots_cache[robots_txt_url(current_uri)] = robots_file if robots_cache
-          if robots_allowed?(robots_file, useragent, current_uri)
-            h = EM::HttpRequest.new(current_uri).get(options)
-            h.callback(&_handle_uri_callback)
-            h.errback {
-              @http_client = h
-              @error = h.error
+          robots_cache[robots_url.to_s] = robots_file if robots_cache
+          if robots_allowed?(robots_file, useragent, robots_url, current_uri)
+            begin
+              h = EM::HttpRequest.new(current_uri).get(options)
+              h.callback(&_handle_uri_callback)
+              h.errback {
+                @http_client = h
+                @error = h.error
+                fail(self)
+              }
+            rescue StandardError => se
+              @http_client = EM::HttpClient.new("")
+              @error = "#{se.message}\n#{se.backtrace.inspect}"
               fail(self)
-            }
+            end
           else
             @http_client = EM::HttpClient.new("")
             @error = "robots denied"
@@ -101,7 +116,7 @@ class RDaneel
           end
         }
         robots.errback {
-          robots_cache.put[robots_txt_url(current_uri)] = "" if robots_cache
+          robots_cache[robots_url.to_s] = "" if robots_cache
           h = EM::HttpRequest.new(current_uri).get(options)
           h.callback(&_handle_uri_callback)
           h.errback {
@@ -121,10 +136,14 @@ class RDaneel
 
   protected
 
-  def robots_allowed?(robots_file, useragent, u)
-    rules = RobotRules.new(useragent)
-    rules.parse(u.to_s, robots_file)
-    rules.allowed? u.to_s
+  def robots_allowed?(robots_file, useragent, robots_url, url)
+    begin
+      rules = RobotRules.new(useragent)
+      rules.parse(robots_url, robots_file)
+      return rules.allowed? url
+    rescue StandardError => err
+      return true
+    end
   end
 
   def robots_txt_url(u)
@@ -133,7 +152,7 @@ class RDaneel
     else
       "#{u.host}:#{u.port}"
     end
-    "http://#{location}/robots.txt"
+    Addressable::URI.parse("http://#{location}/robots.txt")
   end
 
   def success?(http_client)
@@ -146,8 +165,9 @@ class RDaneel
 
   def redirect_url(http_client, u)
     location = Addressable::URI.parse(http_client.response_header.location)
-    return u.join(location) if location.relative?
-    return location
+    location = u.join(location) if location.relative?
+    location.path = "/" if location.path.nil? || location.path == ""
+    location
   end
 end
 

data/spec/no_redirects_neither_robots_spec.rb

@@ -2,6 +2,41 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "RDaneel when there are no redirects" do
 
+  describe "when not exist a robots.txt (404) and the url requested is /" do
+    before(:each) do
+      burrito.mount( :path  => '/', :status => 200,
+                        :body  => 'Hello World!', :block  => should_be_hit_once )
+      burrito.mount( :path  => '/robots.txt',  :status => 404,
+                        :block => should_be_hit_once )
+    end
+
+    after(:each) do
+      burrito.unmount('/')
+      burrito.unmount('/robots.txt')
+    end
+
+    it "should get the content is the url not end with /" do
+
+      EM.run do
+        r = RDaneel.new("http://127.0.0.1:8080")
+        r.callback do
+          r.http_client.response_header.status.should == 200
+          r.http_client.response.should == "Hello World!"
+          r.redirects.should be_empty
+          EM.stop
+        end
+        r.errback do
+          fail
+          EM.stop
+        end
+        r.get
+      end
+
+    end
+  end
+
+
+
   describe "when a successfull status different than 200 is issued for robots.txt" do
 
     it "should get the content ignoring the redirect"

data/spec/spec_helper.rb

@@ -3,7 +3,6 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rubygems'
 require 'rdaneel'
 require 'spec'
-require 'spec/autorun'
 require 'webrick'
 
 Spec::Runner.configure do |config|
@@ -33,6 +32,7 @@ class Burrito
       @server.start
     }
     @server
+    sleep 0.5
   end
 
   def mount( opts )

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Edgar Gonzalez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-19 00:00:00 -04:30
+date: 2010-07-21 00:00:00 -04:30
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -42,8 +42,8 @@ dependencies:
         segments: 
         - 0
         - 9
-        - 1
-        version: 0.9.1
+        - 3
+        version: 0.9.3
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency