RubyGems - rdaneel - Versions diffs - 0.0.0 → 0.1.0 - Mend

rdaneel 0.0.0 → 0.1.0

Files changed (11) hide show

data/LICENSE +2 -1
data/README.rdoc +66 -5
data/Rakefile +2 -2
data/VERSION +1 -1
data/lib/rdaneel.rb +117 -57
data/spec/no_redirects_neither_robots_spec.rb +136 -0
data/spec/redirects_without_robots_spec.rb +175 -0
data/spec/spec_helper.rb +67 -0
data/spec/using_cache_spec.rb +44 -0
metadata +13 -6
data/spec/rdaneel_spec.rb +0 -96

data/LICENSE CHANGED Viewed

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Edgar
+Copyright (c) 2009 has_many :deveopers
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -18,3 +18,4 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc CHANGED Viewed

@@ -1,13 +1,74 @@
 = R.Daneel
-Obey robots.txt on top of em-http-request (http://github.com/igrigorik/em-http-request - Asynchronous HTTP Client)
+An EventMachine+Ruby library to fetch urls following robots.txt rules.
+RDaneel is built it on top of @igrigorik's {em-http-request}[http://github.com/igrigorik/em-http-request]
+== Features
+- Support following redirects, honoring robots.txt for each host in the redirect chain.
+- Support an external cache to store robots.txt
+- Compatible with all options defined in em-http-request
+== Install
+  $ gem install rdaneel
+== Examples
+=== Following redirects
+  require 'rdaneel'
+  EM.run do
+    r = RDaneel.new("http://bit.ly/cbEnpa")
+    r.callback{
+      puts r.http_client.response_header.status
+      puts r.http_client.response[0,80]
+      puts r.redirects
+      puts r.uri
+      EM.stop
+    }
+    r.errback{
+      puts "should not happen"
+      EM.stop
+    }
+    r.get(:redirects => 3)
+  end
+  => 200
+  => <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  => http://bit.ly:80/cbEnpa
+  => http://github.com:80/hasmanydevelopers/RDaneel
+=== Denied by robots.txt
+  require 'rdaneel'
+  EM.run do
+    r = RDaneel.new("http://github.com/hasmanydevelopers/RDaneel/tarball/v0.0.0")
+    r.callback{
+      puts "should not happen"
+      EM.stop
+    }
+    r.errback{
+      puts r.error
+      EM.stop
+    }
+    r.get(:redirects => 3)
+  end
+  => robots denied
+== Why RDaneel?
 R Daneel Olivaw is a fictional robot created by Isaac Asimov - http://en.wikipedia.org/wiki/R._Daneel_Olivaw
-== Important
+== Acknowledge
-The same em-http-request options apply here.
-But when following redirects the method won't check the intermediate robots.txt, just the first one.
+To Ilya Grigorik (@igrigorik) for em-http-request lib and his support and advice.
 == Note on Patches/Pull Requests
@@ -22,5 +83,5 @@ But when following redirects the method won't check the intermediate robots.txt,
 == Copyright
-Copyright (c) 2010 Edgar. See LICENSE for details.
+Copyright (c) 2010 has_many :developers. See LICENSE for details.

data/Rakefile CHANGED Viewed

@@ -7,9 +7,9 @@ begin
     gem.name = "rdaneel"
     gem.summary = %Q{Obey robots.txt on top of em-http-request (Asynchronous HTTP Client)}
     gem.description = %Q{Add robots.txt support on top of em-http-request}
-    gem.email = "edgargonzalez@gmail.com"
+    gem.email = ["edgargonzalez@gmail.com", "anibalrojas@gmail.com"]
     gem.homepage = "http://github.com/hasmanydevelopers/RDaneel"
-    gem.authors = ["Edgar Gonzalez"]
+    gem.authors = ["Edgar Gonzalez", "Anibal Rojas"]
     gem.add_dependency("em-http-request", ">= 0.2.10")
     gem.add_dependency('robot_rules', '>= 0.9.1')
     gem.add_development_dependency "rspec", ">= 1.2.9"

data/VERSION CHANGED Viewed

	@@ -1 +1 @@
1	- 0.0.0
1	+ 0.1.0

data/lib/rdaneel.rb CHANGED Viewed

@@ -1,16 +1,14 @@
 require 'em-http'
 require 'robot_rules'
-require 'net/http'
-module Net
-  class DisobeyingRobotsTxt < HTTPBadResponse ; end
-end
 class RDaneel
+  include EM::Deferrable
+  DEFAULT_OPTIONS = {:head => {'user-agent' => 'RDaneel'}}
   class << self
-    def robots_cache=(klass, options={})
-      @robots_cache = klass.new(options)
+    def robots_cache=(c)
+      @robots_cache = c
     end
     def robots_cache
@@ -18,76 +16,138 @@ class RDaneel
     end
   end
+  attr_accessor :uri
+  attr_reader :error, :redirects, :http_client
   def initialize(uri)
     @uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri)
+    @redirects = []
   end
-  def robots_cache
-    self.class.robots_cache
-  end
+  def get(opts = {})
+    current_uri = @uri
+    options = DEFAULT_OPTIONS.merge(opts)
+    max_redirects = options.delete(:redirects).to_i
+    useragent = options[:head]['user-agent']
-  #
-  # The same em-http-request options apply here.
-  # But when following redirects the method won't check the intermediate robots.txt, just the first one.
-  #
-  def get(options = {}, &blk)
-    useragent = "RDaneel"
-    if options[:head]
-      options[:head].keys.each do |k|
-        useragent = options[:head][k] if k.to_s.downcase == "user-agent"
-      end
-    end
-    if robots_cache && robots_file = robots_cache.get(robots_txt_url)
-      if robots_allowed?(robots_file, useragent)
-        http = EventMachine::HttpRequest.new(@uri).get(options)
-        http.callback {blk.call(http)}
-        http.errback {blk.call(http)}
+    _get = lambda {}
+    _handle_uri_callback = lambda {|h|
+      if success?(h)
+        @uri = current_uri if current_uri != @uri
+        @http_client = h
+        succeed(self)
+      elsif redirected?(h)
+        if @redirects.size >= max_redirects
+          @http_client = h
+          @error = "Exceeded maximum number of redirects"
+          fail(self)
+          return
+        end
+        begin
+          @redirects << current_uri.to_s
+          current_uri = redirect_url(h, current_uri)
+          if @redirects.include?(current_uri.to_s)
+            @http_client = h
+            @error = "infinite redirect"
+            fail(self)
+            return
+          end
+          _get.call
+        rescue
+          @http_client = h
+          @error = "mal formed redirected url"
+          fail(self)
+        end
       else
-        conn = EventMachine::HttpClient.new("")
-        conn.uri = @uri
-        conn.on_error("robots.txt")
-        blk.call(conn)
+        # other error
+        self.http_client = h
+        @error = "not success and not redirect"
+        fail(self)
       end
-    else
-      robots = EventMachine::HttpRequest.new(robots_txt_url).get
-      robots.callback {
-        robots_file = robots.response
-        robots_cache.put(robots_txt_url, robots_file) if robots_cache
-        if robots_allowed?(robots_file, useragent)
-          http = EventMachine::HttpRequest.new(@uri).get(options)
-          http.callback {blk.call(http)}
-          http.errback {blk.call(http)}
+    }
+    _get = lambda {
+      if robots_cache && robots_file = robots_cache[robots_txt_url(current_uri)]
+        if robots_allowed?(robots_file, useragent, current_uri)
+          h = EM::HttpRequest.new(current_uri).get(options)
+          h.callback(&_handle_uri_callback)
+          h.errback {
+            @http_client = h
+            @error = h.error
+            fail(self)
+          }
         else
-          conn = EventMachine::HttpClient.new("")
-          conn.uri = @uri
-          conn.on_error("robots.txt")
-          blk.call(conn)
+          @http_client = EM::HttpClient.new("")
+          @error = "robots denied"
+          fail(self)
         end
-      }
-      robots.errback {
-        http = EventMachine::HttpRequest.new(@uri).get(options)
-        http.callback {blk.call(http)}
-        http.errback {blk.call(http)}
-      }
-    end
+      else
+        robots = EM::HttpRequest.new(robots_txt_url(current_uri)).get
+        robots.callback {
+          robots_file = robots.response
+          robots_cache[robots_txt_url(current_uri)] = robots_file if robots_cache
+          if robots_allowed?(robots_file, useragent, current_uri)
+            h = EM::HttpRequest.new(current_uri).get(options)
+            h.callback(&_handle_uri_callback)
+            h.errback {
+              @http_client = h
+              @error = h.error
+              fail(self)
+            }
+          else
+            @http_client = EM::HttpClient.new("")
+            @error = "robots denied"
+            fail(self)
+          end
+        }
+        robots.errback {
+          robots_cache.put[robots_txt_url(current_uri)] = "" if robots_cache
+          h = EM::HttpRequest.new(current_uri).get(options)
+          h.callback(&_handle_uri_callback)
+          h.errback {
+            @http_client = h
+            @error = h.error
+            fail(self)
+          }
+        }
+      end
+    }
+    _get.call
+  end
+  def robots_cache
+    self.class.robots_cache
   end
   protected
-  def robots_allowed?(robots_file, useragent)
+  def robots_allowed?(robots_file, useragent, u)
     rules = RobotRules.new(useragent)
-    rules.parse(@uri.to_s, robots_file)
-    rules.allowed? @uri.to_s
+    rules.parse(u.to_s, robots_file)
+    rules.allowed? u.to_s
   end
-  def robots_txt_url
-    location = if @uri.port == 80
-      @uri.host
+  def robots_txt_url(u)
+    location = if u.port == 80
+      u.host
     else
-      "#{@uri.host}:#{@uri.port}"
+      "#{u.host}:#{u.port}"
     end
     "http://#{location}/robots.txt"
   end
+  def success?(http_client)
+    http_client.response_header.status == 200
+  end
+  def redirected?(http_client)
+    http_client.response_header.status == 301 || http_client.response_header.status == 302
+  end
+  def redirect_url(http_client, u)
+    location = Addressable::URI.parse(http_client.response_header.location)
+    return u.join(location) if location.relative?
+    return location
+  end
 end

data/spec/no_redirects_neither_robots_spec.rb ADDED Viewed

@@ -0,0 +1,136 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe "RDaneel when there are no redirects" do
+  describe "when a successfull status different than 200 is issued for robots.txt" do
+    it "should get the content ignoring the redirect"
+  end
+  describe "when a redirect other than 301 and 302 is issued for robots.txt" do
+    it "should get the content ignoring the redirect"
+  end
+  (301..302).each do |status|
+    describe "when robots.txt has been moved (http code #{status})" do
+      before(:each) do
+        burrito.mount( :path  => '/hello_world', :status => 200,
+                        :body  => 'Hello World!', :block  => should_be_hit_once )
+        burrito.mount( :path  => '/robots.txt',  :status => status,
+                        :location => 'http://127.0.0.1:8080/golems.txt',
+                        :block => should_be_hit_once )
+        burrito.mount( :path  => '/golems.txt',  :status => 200,
+                        :block => should_not_be_hit )
+      end
+      after(:each) do
+        burrito.unmount('/hello_world')
+        burrito.unmount('/robots.txt')
+        burrito.unmount('/golems.txt')
+      end
+      it "should get the content ignoring the redirect" do
+        EM.run do
+          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r.callback do
+            r.http_client.response_header.status.should == 200
+            r.http_client.response.should == "Hello World!"
+            r.redirects.should be_empty
+            EM.stop
+          end
+          r.errback do
+            fail
+            EM.stop
+          end
+          r.get
+        end
+      end
+    end
+  end
+  (400..417).each do |status|
+    describe "when there is a CLIENT error #{status} associated to robots.txt" do
+      before(:each) do
+        burrito.mount( :path  => '/hello_world', :status => 200,
+                          :body  => 'Hello World!', :block  => should_be_hit_once )
+        burrito.mount( :path  => '/robots.txt',  :status => status,
+                          :block => should_be_hit_once )
+      end
+      after(:each) do
+        burrito.unmount('/hello_world')
+        burrito.unmount('/robots.txt')
+      end
+      it "should get the content" do
+        EM.run do
+          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r.callback do
+            r.http_client.response_header.status.should == 200
+            r.http_client.response.should == "Hello World!"
+            r.redirects.should be_empty
+            EM.stop
+          end
+          r.errback do
+            fail
+            EM.stop
+          end
+          r.get
+        end
+      end
+    end
+  end
+  (500..505).each do |status|
+    describe "when there is a SERVER error #{status} associated to robots.txt" do
+      before(:each) do
+        burrito.mount( :path  => '/hello_world', :status => 200,
+                        :body  => 'Hello World!', :block  => should_be_hit_once )
+        burrito.mount( :path  => '/robots.txt',  :status => status,
+                        :block => should_be_hit_once )
+      end
+      after (:each) do
+        burrito.unmount('/hello_world')
+        burrito.unmount('/robots.txt')
+      end
+      it "should get the content" do
+        EM.run do
+          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r.callback do
+            r.http_client.response_header.status.should == 200
+            r.http_client.response.should == "Hello World!"
+            r.redirects.should be_empty
+            EM.stop
+          end
+          r.errback do
+            fail
+            EM.stop
+          end
+          r.get
+        end
+      end
+    end
+  end
+end

data/spec/redirects_without_robots_spec.rb ADDED Viewed

@@ -0,0 +1,175 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe "RDaneel when there are redirects" do
+  describe "when there is no robots.txt in the host (ONLY one host)" do
+    describe "when no redirection limit has been set" do
+      before(:each) do
+        burrito.mount( :path  => '/robots.txt',  :status => 404,
+                        :block => should_be_hit_once )
+        burrito.mount( :path  => '/redirect_me', :status => 301,
+                        :location  => 'http://127.0.0.1:8080/hello_world',
+                        :block  => should_be_hit_once )
+        burrito.mount( :path  => '/hello_world', :status => 200,
+                        :body  => 'Hello World!',
+                        :block  => should_not_be_hit )
+      end
+      after(:each) do
+        burrito.unmount('/robots.txt')
+        burrito.unmount('/redirect_me')
+        burrito.unmount('/hello_world')
+      end
+      it "should not follow redirects" do
+        EM.run do
+          r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+          r.callback do
+            fail
+            EM.stop
+          end
+          r.errback do
+            r.redirects.should be_empty
+            r.error.should == "Exceeded maximum number of redirects"
+            EM.stop
+          end
+          r.get
+        end
+      end
+    end
+    describe "when a maximum number or redirects is set" do
+      describe "when there are less redirects than the maximum specified" do
+        before(:each) do
+          burrito.mount( :path  => '/robots.txt',  :status => 404,
+                          :block => should_be_hit(3) )
+          burrito.mount( :path  => '/redirect_me', :status => 301,
+                          :location  => 'http://127.0.0.1:8080/redirect_me_again',
+                          :block  => should_be_hit_once )
+          burrito.mount( :path  => '/redirect_me_again', :status => 301,
+                          :location  => 'http://127.0.0.1:8080/hello_world',
+                          :block  => should_be_hit_once )
+          burrito.mount( :path  => '/hello_world', :status => 200,
+                          :body  => 'Hello World!',
+                          :block  => should_be_hit_once )
+        end
+        after(:each) do
+          burrito.unmount('/robots.txt')
+          burrito.unmount('/redirect_me')
+          burrito.unmount('/redirect_me_again')
+          burrito.unmount('/hello_world')
+        end
+        it "should get the content following all the redirects" do
+          EM.run do
+            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r.callback do
+              r.http_client.response_header.status.should == 200
+              r.http_client.response.should == "Hello World!"
+              r.redirects.should == [ "http://127.0.0.1:8080/redirect_me",
+                                      "http://127.0.0.1:8080/redirect_me_again"]
+              r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+              EM.stop
+            end
+            r.errback do
+              fail
+              EM.stop
+            end
+            r.get(:redirects => 3)
+          end
+        end
+      end
+      describe "when there are as many redirects as the maximum" do
+        before(:each) do
+          burrito.mount( :path  => '/robots.txt',  :status => 404,
+                          :block => should_be_hit_twice )
+          burrito.mount( :path  => '/redirect_me', :status => 301,
+                          :location  => 'http://127.0.0.1:8080/hello_world',
+                          :block  => should_be_hit_once )
+          burrito.mount( :path  => '/hello_world', :status => 200,
+                          :body  => 'Hello World!',
+                          :block  => should_be_hit_once )
+        end
+        after(:each) do
+          burrito.unmount('/robots.txt')
+          burrito.unmount('/redirect_me')
+          burrito.unmount('/hello_world')
+        end
+        it "should get the content following all the redirects" do
+          EM.run do
+            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r.callback do
+              r.http_client.response_header.status.should == 200
+              r.http_client.response.should == "Hello World!"
+              r.redirects.should == ['http://127.0.0.1:8080/redirect_me']
+              r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+              EM.stop
+            end
+            r.errback do
+              fail
+              EM.stop
+            end
+            r.get(:redirects => 1)
+          end
+        end
+      end
+      describe "when the number of redirects exceed the maximum specified" do
+        before(:each) do
+          burrito.mount( :path  => '/robots.txt',  :status => 404,
+                          :block => should_be_hit_twice )
+          burrito.mount( :path  => '/redirect_me', :status => 301,
+                          :location  => 'http://127.0.0.1:8080/redirect_me_again',
+                          :block  => should_be_hit_once )
+          burrito.mount( :path  => '/redirect_me_again', :status => 301,
+                          :location  => 'http://127.0.0.1:8080/hello_world',
+                          :block  => should_be_hit_once )
+          burrito.mount( :path  => '/hello_world', :status => 200,
+                          :body  => 'Hello World!',
+                          :block  => should_not_be_hit )
+        end
+        after(:each) do
+          burrito.unmount('/robots.txt')
+          burrito.unmount('/redirect_me')
+          burrito.unmount('/redirect_me_again')
+          burrito.unmount('/hello_world')
+        end
+        it "should stop following redirects once the  maximum specified is reached" do
+          EM.run do
+            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r.callback do
+              fail
+              EM.stop
+            end
+            r.errback do
+              r.redirects.should == ['http://127.0.0.1:8080/redirect_me']
+              r.error.should == "Exceeded maximum number of redirects"
+              EM.stop
+            end
+            r.get(:redirects => 1)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb CHANGED Viewed

@@ -4,8 +4,75 @@ require 'rubygems'
 require 'rdaneel'
 require 'spec'
 require 'spec/autorun'
+require 'webrick'
 Spec::Runner.configure do |config|
+  config.before :suite do
+    burrito
+  end
+  config.after :suite do
+    burrito.stop
+  end
+end
+def burrito
+  Thread.current[:burrito] ||= Burrito.new
+end
+class Burrito
+  def initialize( options={}, &blk )
+    webrick_log_file = '/dev/null'  # disable logging
+    webrick_logger = WEBrick::Log.new(webrick_log_file, WEBrick::Log::DEBUG)
+    access_log_stream = webrick_logger
+    access_log = [[ access_log_stream, WEBrick::AccessLog::COMBINED_LOG_FORMAT ]]
+    default_opts = {:Port => 8080, :Logger => webrick_logger, :AccessLog => access_log }
+    @server = WEBrick::HTTPServer.new( default_opts.merge(options) )
+    @server_thread = Thread.new {
+      blk.call(@server) if blk
+      @server.start
+    }
+    @server
+  end
+  def mount( opts )
+    raise ":path is required" if opts[:path].nil?
+    raise ":status is required" if opts[:status].nil?
+    @server.mount_proc( opts[:path],
+      lambda { |req, resp|
+               resp.status = opts[:status]
+               resp.body = opts[:body] unless opts[:body].nil?
+               resp['Location'] = opts[:location] unless opts[:location].nil?
+               opts[:block].call unless opts[:block].nil?
+             } )
+  end
+  def stop
+    @server.shutdown
+    @server_thread.join
+  end
+  def unmount(path)
+    @server.unmount(path)
+  end
+end
+def should_not_be_hit
+  should_be_hit( 0 )
+end
+def should_be_hit_once
+  should_be_hit( 1 )
+end
+def should_be_hit_twice
+  should_be_hit( 2 )
+end
+def should_be_hit( times = 1 )
+  l = lambda {}
+  m = l.should_receive(:call).exactly(times).times
+  return l
 end

data/spec/using_cache_spec.rb ADDED Viewed

@@ -0,0 +1,44 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe "RDaneel when there is a cache" do
+  describe "when there is no robots.txt in the host" do
+    before(:each) do
+      RDaneel.robots_cache = {}
+      burrito.mount( :path  => '/robots.txt',  :status => 404,
+                      :block => should_be_hit_once )
+      burrito.mount( :path  => '/redirect_me', :status => 301,
+                      :location  => 'http://127.0.0.1:8080/hello_world',
+                      :block  => should_be_hit_once )
+      burrito.mount( :path  => '/hello_world', :status => 200,
+                      :body  => 'Hello World!',
+                      :block  => should_be_hit_once )
+    end
+    after(:each) do
+      burrito.unmount('/robots.txt')
+      burrito.unmount('/redirect_me')
+      burrito.unmount('/hello_world')
+    end
+    it "should try to get the robots.txt just once" do
+      EM.run do
+        r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+        r.callback do
+          r.http_client.response_header.status.should == 200
+          r.http_client.response.should == "Hello World!"
+          r.redirects.should == [ "http://127.0.0.1:8080/redirect_me"]
+          r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+          EM.stop
+        end
+        r.errback do
+          fail
+          EM.stop
+        end
+        r.get(:redirects => 3)
+      end
+    end
+  end
+end

metadata CHANGED Viewed

@@ -4,17 +4,18 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments:
   - 0
+  - 1
   - 0
-  - 0
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Edgar Gonzalez
+- Anibal Rojas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2010-07-14 00:00:00 -04:30
+date: 2010-07-19 00:00:00 -04:30
 default_executable:
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -60,7 +61,9 @@ dependencies:
   type: :development
   version_requirements: *id003
 description: Add robots.txt support on top of em-http-request
-email: edgargonzalez@gmail.com
+email:
+- edgargonzalez@gmail.com
+- anibalrojas@gmail.com
 executables: []
 extensions: []
@@ -76,9 +79,11 @@ files:
 - Rakefile
 - VERSION
 - lib/rdaneel.rb
-- spec/rdaneel_spec.rb
+- spec/no_redirects_neither_robots_spec.rb
+- spec/redirects_without_robots_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/using_cache_spec.rb
 has_rdoc: true
 homepage: http://github.com/hasmanydevelopers/RDaneel
 licenses: []
@@ -110,5 +115,7 @@ signing_key:
 specification_version: 3
 summary: Obey robots.txt on top of em-http-request (Asynchronous HTTP Client)
 test_files:
+- spec/using_cache_spec.rb
+- spec/no_redirects_neither_robots_spec.rb
 - spec/spec_helper.rb
-- spec/rdaneel_spec.rb
+- spec/redirects_without_robots_spec.rb

data/spec/rdaneel_spec.rb DELETED Viewed

@@ -1,96 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-require 'webrick'
-describe "RDaneel" do
-  describe "when there is no robots.txt" do
-    before(:all) do
-      start_server do |s|
-        s.mount_proc('/hello_world', lambda { |req, resp| resp.status = 200; resp.body = "hello world"})
-      end
-    end
-    after(:all) do
-      stop_server
-    end
-    it "should follow and get the uri" do
-      EM.run {
-        RDaneel.new("http://127.0.0.1:8080/hello_world").get do |http|
-          http.response_header.status.should == 200
-          http.response.should == "hello world"
-          http.error.should == ''
-          EM.stop
-        end
-      }
-    end
-  end
-  describe "when there is a robots.txt that allow the uri requested" do
-    before(:all) do
-      start_server do |s|
-        s.mount_proc('/robots.txt', lambda { |req, resp| resp.status = 200; resp.body = "User-agent: *\nDisallow: /images"})
-        s.mount_proc('/hello_world', lambda { |req, resp| resp.status = 200; resp.body = "hello world"})
-      end
-    end
-    after(:all) do
-      stop_server
-    end
-    it "should follow and get the uri" do
-      EM.run {
-        RDaneel.new("http://127.0.0.1:8080/hello_world").get do |http|
-          http.response_header.status.should == 200
-          http.response.should == "hello world"
-          http.error.should == ''
-          EM.stop
-        end
-      }
-    end
-  end
-  describe "when there is a robots.txt that disallow all content for all bots" do
-    before(:all) do
-      start_server do |s|
-        s.mount_proc('/robots.txt', lambda { |req, resp| resp.status = 200; resp.body = "User-agent: *\nDisallow: /"})
-        s.mount_proc('/hello_world', lambda { |req, resp| resp.status = 200; resp.body = "hello world"})
-      end
-    end
-    after(:all) do
-      stop_server
-    end
-    it "shouldn't get the uri" do
-      EM.run {
-          RDaneel.new("http://127.0.0.1:8080/hello_world").get do |http|
-          http.error.should == 'robots.txt'
-          http.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
-          EM.stop
-        end
-      }
-    end
-  end
-end
-def start_server(options={}, &blk)
-  @server = WEBrick::HTTPServer.new({:Port => 8080}.merge(options))
-  @server_thread = Thread.new {
-    blk.call(@server) if blk
-    @server.start
-  }
-end
-def stop_server
-  @server.shutdown
-  @server_thread.join
-end
-def robots_txt
-end