rdaneel 0.1.2 → 0.1.3

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.3

data/lib/rdaneel.rb

@@ -91,7 +91,7 @@ class RDaneel
         end
       else
         robots_url = robots_txt_url(current_uri)
-        robots = EM::HttpRequest.new(robots_url).get
+        robots = EM::HttpRequest.new(robots_url).get(:redirects => 2) # get the robots.txt following redirects
         robots.callback {
           robots_file = robots.response
           robots_cache[robots_url.to_s] = robots_file if robots_cache

data/spec/no_redirects_neither_robots_spec.rb

@@ -2,76 +2,38 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "RDaneel when there are no redirects" do
 
-  describe "when not exist a robots.txt (404) and the url requested is /" do
-    before(:each) do
-      burrito.mount( :path  => '/', :status => 200,
-                        :body  => 'Hello World!', :block  => should_be_hit_once )
-      burrito.mount( :path  => '/robots.txt',  :status => 404,
-                        :block => should_be_hit_once )
-    end
-
-    after(:each) do
-      burrito.unmount('/')
-      burrito.unmount('/robots.txt')
-    end
-
-    it "should get the content is the url not end with /" do
-
-      EM.run do
-        r = RDaneel.new("http://127.0.0.1:8080")
-        r.callback do
-          r.http_client.response_header.status.should == 200
-          r.http_client.response.should == "Hello World!"
-          r.redirects.should be_empty
-          EM.stop
-        end
-        r.errback do
-          fail
-          EM.stop
-        end
-        r.get
-      end
-
-    end
-  end
-
-
+  let(:port) {8083}
 
   describe "when a successfull status different than 200 is issued for robots.txt" do
-
     it "should get the content ignoring the redirect"
-
   end
 
   describe "when a redirect other than 301 and 302 is issued for robots.txt" do
-
     it "should get the content ignoring the redirect"
-
   end
 
   (301..302).each do |status|
 
     describe "when robots.txt has been moved (http code #{status})" do
       before(:each) do
-        burrito.mount( :path  => '/hello_world', :status => 200,
+        server_setup(port+status) do |server|
+          mount(server, :path  => '/hello_world', :status => 200,
                         :body  => 'Hello World!', :block  => should_be_hit_once )
-        burrito.mount( :path  => '/robots.txt',  :status => status,
-                        :location => 'http://127.0.0.1:8080/golems.txt',
+          mount(server, :path  => '/robots.txt',  :status => status,
+                        :location => "http://127.0.0.1:#{port+status}/golems.txt",
+                        :block => should_be_hit_once )
+          mount(server, :path  => '/golems.txt',  :status => 200,
                         :block => should_be_hit_once )
-        burrito.mount( :path  => '/golems.txt',  :status => 200,
-                        :block => should_not_be_hit )
+        end
       end
 
       after(:each) do
-        burrito.unmount('/hello_world')
-        burrito.unmount('/robots.txt')
-        burrito.unmount('/golems.txt')
+        server_shutdown
       end
 
-      it "should get the content ignoring the redirect" do
-
+      it "should get the redirected robots.txt and the content" do
         EM.run do
-          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r = RDaneel.new("http://127.0.0.1:#{port+status}/hello_world")
           r.callback do
             r.http_client.response_header.status.should == 200
             r.http_client.response.should == "Hello World!"
@@ -84,7 +46,6 @@ describe "RDaneel when there are no redirects" do
           end
           r.get
         end
-
       end
 
     end
@@ -95,21 +56,21 @@ describe "RDaneel when there are no redirects" do
 
     describe "when there is a CLIENT error #{status} associated to robots.txt" do
       before(:each) do
-        burrito.mount( :path  => '/hello_world', :status => 200,
-                          :body  => 'Hello World!', :block  => should_be_hit_once )
-        burrito.mount( :path  => '/robots.txt',  :status => status,
-                          :block => should_be_hit_once )
+        server_setup(port+status) do |server|
+          mount(server, :path  => '/hello_world', :status => 200,
+                        :body  => 'Hello World!', :block  => should_be_hit_once )
+          mount(server, :path  => '/robots.txt',  :status => status,
+                        :block => should_be_hit_once )
+        end
       end
 
       after(:each) do
-        burrito.unmount('/hello_world')
-        burrito.unmount('/robots.txt')
+        server_shutdown
       end
 
       it "should get the content" do
-
         EM.run do
-          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r = RDaneel.new("http://127.0.0.1:#{port+status}/hello_world")
           r.callback do
             r.http_client.response_header.status.should == 200
             r.http_client.response.should == "Hello World!"
@@ -122,7 +83,6 @@ describe "RDaneel when there are no redirects" do
           end
           r.get
         end
-
       end
 
     end
@@ -133,21 +93,21 @@ describe "RDaneel when there are no redirects" do
 
     describe "when there is a SERVER error #{status} associated to robots.txt" do
       before(:each) do
-        burrito.mount( :path  => '/hello_world', :status => 200,
+        server_setup(port+status) do |server|
+          mount(server, :path  => '/hello_world', :status => 200,
                         :body  => 'Hello World!', :block  => should_be_hit_once )
-        burrito.mount( :path  => '/robots.txt',  :status => status,
+          mount(server, :path  => '/robots.txt',  :status => status,
                         :block => should_be_hit_once )
+        end
       end
 
       after (:each) do
-        burrito.unmount('/hello_world')
-        burrito.unmount('/robots.txt')
+        server_shutdown
       end
 
       it "should get the content" do
-
         EM.run do
-          r = RDaneel.new("http://127.0.0.1:8080/hello_world")
+          r = RDaneel.new("http://127.0.0.1:#{port+status}/hello_world")
           r.callback do
             r.http_client.response_header.status.should == 200
             r.http_client.response.should == "Hello World!"
@@ -160,7 +120,6 @@ describe "RDaneel when there are no redirects" do
           end
           r.get
         end
-
       end
 
     end

data/spec/redirects_without_robots_spec.rb

@@ -2,29 +2,31 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "RDaneel when there are redirects" do
 
+  let(:port) {8081}
+
   describe "when there is no robots.txt in the host (ONLY one host)" do
 
     describe "when no redirection limit has been set" do
       before(:each) do
-        burrito.mount( :path  => '/robots.txt',  :status => 404,
+        server_setup(port) do |server|
+          mount(server, :path  => '/robots.txt',  :status => 404,
                         :block => should_be_hit_once )
-        burrito.mount( :path  => '/redirect_me', :status => 301,
-                        :location  => 'http://127.0.0.1:8080/hello_world',
+          mount(server, :path  => '/redirect_me', :status => 301,
+                        :location  => "http://127.0.0.1:#{port}/hello_world",
                         :block  => should_be_hit_once )
-        burrito.mount( :path  => '/hello_world', :status => 200,
+          mount(server, :path  => '/hello_world', :status => 200,
                         :body  => 'Hello World!',
                         :block  => should_not_be_hit )
+        end
       end
 
       after(:each) do
-        burrito.unmount('/robots.txt')
-        burrito.unmount('/redirect_me')
-        burrito.unmount('/hello_world')
+        server_shutdown
       end
 
       it "should not follow redirects" do
         EM.run do
-          r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+          r = RDaneel.new("http://127.0.0.1:#{port}/redirect_me")
           r.callback do
             fail
             EM.stop
@@ -45,35 +47,34 @@ describe "RDaneel when there are redirects" do
 
       describe "when there are less redirects than the maximum specified" do
         before(:each) do
-          burrito.mount( :path  => '/robots.txt',  :status => 404,
+          server_setup(port) do |server|
+            mount(server, :path  => '/robots.txt',  :status => 404,
                           :block => should_be_hit(3) )
-          burrito.mount( :path  => '/redirect_me', :status => 301,
-                          :location  => 'http://127.0.0.1:8080/redirect_me_again',
+            mount(server, :path  => '/redirect_me', :status => 301,
+                          :location  => "http://127.0.0.1:#{port}/redirect_me_again",
                           :block  => should_be_hit_once )
-          burrito.mount( :path  => '/redirect_me_again', :status => 301,
-                          :location  => 'http://127.0.0.1:8080/hello_world',
+            mount(server, :path  => '/redirect_me_again', :status => 301,
+                          :location  => "http://127.0.0.1:#{port}/hello_world",
                           :block  => should_be_hit_once )
-          burrito.mount( :path  => '/hello_world', :status => 200,
+            mount(server, :path  => '/hello_world', :status => 200,
                           :body  => 'Hello World!',
                           :block  => should_be_hit_once )
+          end
         end
 
         after(:each) do
-          burrito.unmount('/robots.txt')
-          burrito.unmount('/redirect_me')
-          burrito.unmount('/redirect_me_again')
-          burrito.unmount('/hello_world')
+          server_shutdown
         end
 
         it "should get the content following all the redirects" do
           EM.run do
-            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r = RDaneel.new("http://127.0.0.1:#{port}/redirect_me")
             r.callback do
               r.http_client.response_header.status.should == 200
               r.http_client.response.should == "Hello World!"
-              r.redirects.should == [ "http://127.0.0.1:8080/redirect_me",
-                                      "http://127.0.0.1:8080/redirect_me_again"]
-              r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+              r.redirects.should == [ "http://127.0.0.1:#{port}/redirect_me",
+                                      "http://127.0.0.1:#{port}/redirect_me_again"]
+              r.uri.to_s.should == "http://127.0.0.1:#{port}/hello_world"
               EM.stop
             end
             r.errback do
@@ -89,30 +90,30 @@ describe "RDaneel when there are redirects" do
 
       describe "when there are as many redirects as the maximum" do
         before(:each) do
-          burrito.mount( :path  => '/robots.txt',  :status => 404,
+          server_setup(port) do |server|
+            mount(server, :path  => '/robots.txt',  :status => 404,
                           :block => should_be_hit_twice )
-          burrito.mount( :path  => '/redirect_me', :status => 301,
-                          :location  => 'http://127.0.0.1:8080/hello_world',
+            mount(server, :path  => '/redirect_me', :status => 301,
+                          :location  => "http://127.0.0.1:#{port}/hello_world",
                           :block  => should_be_hit_once )
-          burrito.mount( :path  => '/hello_world', :status => 200,
+            mount(server, :path  => '/hello_world', :status => 200,
                           :body  => 'Hello World!',
                           :block  => should_be_hit_once )
+          end
         end
 
         after(:each) do
-          burrito.unmount('/robots.txt')
-          burrito.unmount('/redirect_me')
-          burrito.unmount('/hello_world')
+          server_shutdown
         end
 
         it "should get the content following all the redirects" do
           EM.run do
-            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r = RDaneel.new("http://127.0.0.1:#{port}/redirect_me")
             r.callback do
               r.http_client.response_header.status.should == 200
               r.http_client.response.should == "Hello World!"
-              r.redirects.should == ['http://127.0.0.1:8080/redirect_me']
-              r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+              r.redirects.should == ["http://127.0.0.1:#{port}/redirect_me"]
+              r.uri.to_s.should == "http://127.0.0.1:#{port}/hello_world"
               EM.stop
             end
             r.errback do
@@ -128,35 +129,34 @@ describe "RDaneel when there are redirects" do
 
       describe "when the number of redirects exceed the maximum specified" do
         before(:each) do
-          burrito.mount( :path  => '/robots.txt',  :status => 404,
+          server_setup(port) do |server|
+            mount(server, :path  => '/robots.txt',  :status => 404,
                           :block => should_be_hit_twice )
-          burrito.mount( :path  => '/redirect_me', :status => 301,
-                          :location  => 'http://127.0.0.1:8080/redirect_me_again',
+            mount(server, :path  => '/redirect_me', :status => 301,
+                          :location  => "http://127.0.0.1:#{port}/redirect_me_again",
                           :block  => should_be_hit_once )
-          burrito.mount( :path  => '/redirect_me_again', :status => 301,
-                          :location  => 'http://127.0.0.1:8080/hello_world',
+            mount(server, :path  => '/redirect_me_again', :status => 301,
+                          :location  => "http://127.0.0.1:#{port}/hello_world",
                           :block  => should_be_hit_once )
-          burrito.mount( :path  => '/hello_world', :status => 200,
+            mount(server, :path  => '/hello_world', :status => 200,
                           :body  => 'Hello World!',
                           :block  => should_not_be_hit )
+          end
         end
 
         after(:each) do
-          burrito.unmount('/robots.txt')
-          burrito.unmount('/redirect_me')
-          burrito.unmount('/redirect_me_again')
-          burrito.unmount('/hello_world')
+          server_shutdown
         end
 
         it "should stop following redirects once the  maximum specified is reached" do
           EM.run do
-            r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+            r = RDaneel.new("http://127.0.0.1:#{port}/redirect_me")
             r.callback do
               fail
               EM.stop
             end
             r.errback do
-              r.redirects.should == ['http://127.0.0.1:8080/redirect_me']
+              r.redirects.should == ["http://127.0.0.1:#{port}/redirect_me"]
               r.error.should == "Exceeded maximum number of redirects"
               EM.stop
             end

data/spec/spec_helper.rb

@@ -5,57 +5,66 @@ require 'rdaneel'
 require 'spec'
 require 'webrick'
 
-Spec::Runner.configure do |config|
-  config.before :suite do
-    burrito
+# keep webrick quiet
+class ::WEBrick::HTTPServer
+  def access_log(config, req, res)
+    # nop
   end
-  config.after :suite do
-    burrito.stop
+end
+class ::WEBrick::BasicLog
+  def log(level, data)
+    # nop
   end
 end
 
-def burrito
-  Thread.current[:burrito] ||= Burrito.new
+def locked_file
+  File.join(File.dirname(__FILE__),"server_lock-#{@__port}")
 end
 
-class Burrito
+def server_setup(port=8080, &blk)
+  @__port = port
+  if @server.nil? and !File.exist?(locked_file)
+    File.open(locked_file,'w') {|f| f << 'locked' }
+    @server = WEBrick::HTTPServer.new :Port => port
+    blk.call(@server) if blk
+    queue = Queue.new # synchronize the thread startup to the main thread
+    @test_thread = Thread.new { queue << 1; @server.start }
 
-  def initialize( options={}, &blk )
-    webrick_log_file = '/dev/null'  # disable logging
-    webrick_logger = WEBrick::Log.new(webrick_log_file, WEBrick::Log::DEBUG)
-    access_log_stream = webrick_logger
-    access_log = [[ access_log_stream, WEBrick::AccessLog::COMBINED_LOG_FORMAT ]]
-    default_opts = {:Port => 8080, :Logger => webrick_logger, :AccessLog => access_log }
-    @server = WEBrick::HTTPServer.new( default_opts.merge(options) )
-    @server_thread = Thread.new {
-      blk.call(@server) if blk
-      @server.start
-    }
-    @server
-    sleep 0.5
-  end
+    # wait for the queue
+    value = queue.pop
 
-  def mount( opts )
-    raise ":path is required" if opts[:path].nil?
-    raise ":status is required" if opts[:status].nil?
-    @server.mount_proc( opts[:path],
-      lambda { |req, resp|
-               resp.status = opts[:status]
-               resp.body = opts[:body] unless opts[:body].nil?
-               resp['Location'] = opts[:location] unless opts[:location].nil?
-               opts[:block].call unless opts[:block].nil?
-             } )
-  end
+    if !value
+      STDERR.puts "Failed to startup test server!"
+      exit(1)
+    end
 
-  def stop
-    @server.shutdown
-    @server_thread.join
+    trap("INT"){server_shutdown}
+    at_exit{server_shutdown}
   end
+end
 
-  def unmount(path)
-    @server.unmount(path)
+def server_shutdown
+  begin
+    if File.exist?(locked_file)
+      File.unlink locked_file
+      @server.shutdown unless @server.nil?
+      @server = nil
+    end
+  rescue Object => e
+    puts "Error #{__FILE__}:#{__LINE__}\n#{e.message}"
   end
+end
 
+def mount(server, opts)
+  raise ":path is required" if opts[:path].nil?
+  raise ":status is required" if opts[:status].nil?
+  server.mount_proc( opts[:path],
+    lambda { |req, resp|
+             resp.status = opts[:status]
+             resp.body = opts[:body] unless opts[:body].nil?
+             resp['Location'] = opts[:location] unless opts[:location].nil?
+             opts[:block].call unless opts[:block].nil?
+           } )
 end
 
 def should_not_be_hit
@@ -76,3 +85,9 @@ def should_be_hit( times = 1 )
   return l
 end
 
+Spec::Runner.configure do |config|
+  config.before :suite do
+    puts "\e[4mThese specs could take a while, please be patience\e[0m"
+  end
+end
+

data/spec/streamed_content_spec.rb

@@ -0,0 +1,51 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "RDaneel when the content is chunked (digg.com)" do
+
+  # digg.com uses chunked encoding
+  # http://www.digg.com is redirected to http://digg.com
+
+  describe "when the url is not redirected" do
+
+    it "should get the content" do
+      EM.run do
+        r = RDaneel.new("http://digg.com")
+        r.callback do
+          r.http_client.response_header.status.should == 200
+          r.http_client.response.should_not be_empty
+          r.redirects.should be_empty
+          EM.stop
+        end
+        r.errback do
+          fail
+          EM.stop
+        end
+        r.get
+      end
+    end
+
+  end
+
+  describe "when the url is redirected" do
+    it "should get the content" do
+      EM.run do
+        r = RDaneel.new("http://bit.ly:80/aquXKU")
+        r.callback do
+          r.http_client.response_header.status.should == 200
+          r.http_client.response.should_not be_empty
+          r.redirects.should == ['http://bit.ly:80/aquXKU']
+          r.uri.to_s.should == "http://www.relevancesells.com:80/2010/03/12/the-1st-7-seconds-rule-for-an-elevator-pitch/"
+          EM.stop
+        end
+        r.errback do
+          puts "Error: #{r.error}"
+          fail
+          EM.stop
+        end
+        r.get(:redirects => 3)
+      end
+    end
+  end
+
+end
+

data/spec/using_cache_spec.rb

@@ -2,34 +2,36 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "RDaneel when there is a cache" do
 
+  let(:port) {8082}
+
   describe "when there is no robots.txt in the host" do
 
     before(:each) do
       RDaneel.robots_cache = {}
-      burrito.mount( :path  => '/robots.txt',  :status => 404,
+      server_setup(port) do |server|
+        mount(server, :path  => '/robots.txt',  :status => 404,
                       :block => should_be_hit_once )
-      burrito.mount( :path  => '/redirect_me', :status => 301,
-                      :location  => 'http://127.0.0.1:8080/hello_world',
+        mount(server, :path  => '/redirect_me', :status => 301,
+                      :location  => "http://127.0.0.1:#{port}/hello_world",
                       :block  => should_be_hit_once )
-      burrito.mount( :path  => '/hello_world', :status => 200,
+        mount(server, :path  => '/hello_world', :status => 200,
                       :body  => 'Hello World!',
                       :block  => should_be_hit_once )
+      end
     end
 
     after(:each) do
-      burrito.unmount('/robots.txt')
-      burrito.unmount('/redirect_me')
-      burrito.unmount('/hello_world')
+      server_shutdown
     end
 
     it "should try to get the robots.txt just once" do
       EM.run do
-        r = RDaneel.new("http://127.0.0.1:8080/redirect_me")
+        r = RDaneel.new("http://127.0.0.1:#{port}/redirect_me")
         r.callback do
           r.http_client.response_header.status.should == 200
           r.http_client.response.should == "Hello World!"
-          r.redirects.should == [ "http://127.0.0.1:8080/redirect_me"]
-          r.uri.to_s.should == "http://127.0.0.1:8080/hello_world"
+          r.redirects.should == [ "http://127.0.0.1:#{port}/redirect_me"]
+          r.uri.to_s.should == "http://127.0.0.1:#{port}/hello_world"
           EM.stop
         end
         r.errback do

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 2
-  version: 0.1.2
+  - 3
+  version: 0.1.3
 platform: ruby
 authors: 
 - Edgar Gonzalez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-21 00:00:00 -04:30
+date: 2010-08-13 00:00:00 -04:30
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -83,6 +83,7 @@ files:
 - spec/redirects_without_robots_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/streamed_content_spec.rb
 - spec/using_cache_spec.rb
 has_rdoc: true
 homepage: http://github.com/hasmanydevelopers/RDaneel
@@ -119,3 +120,4 @@ test_files:
 - spec/no_redirects_neither_robots_spec.rb
 - spec/spec_helper.rb
 - spec/redirects_without_robots_spec.rb
+- spec/streamed_content_spec.rb