cobweb 1.0.11 → 1.0.12

data/spec/cobweb/crawl_worker_spec.rb

@@ -0,0 +1,250 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+#require 'sidekiq/testing'
+
+describe CrawlWorker, :local_only => true do
+
+  before(:all) do
+    #store all existing resque process ids so we don't kill them afterwards
+    @existing_processes = `ps aux | grep sidekiq | grep -v grep | awk '{print $2}'`.split("\n")
+    puts @existing_processes
+    @existing_processes.should be_empty
+  
+    # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
+    puts "Starting Workers... Please Wait..."
+    `mkdir log`
+    `rm -rf output.log`
+    io = IO.popen("nohup sidekiq -r ./lib/crawl_worker.rb -q crawl_worker > ./log/output.log &")
+    puts "Workers Started."
+  
+  end
+
+  before(:each) do
+    @base_url = "http://localhost:3532/"
+    @base_page_count = 77
+  
+    clear_queues
+  end
+
+  describe "with no crawl limit" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :crawl_limit => nil,
+        :quiet => false,
+        :debug => false,
+        :cache => nil,
+        :queue_system => :sidekiq
+      }
+      @cobweb = Cobweb.new @request
+    end
+  
+    it "should crawl entire site" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      CrawlProcessWorker.queue_size.should == @base_page_count
+    end
+    it "detect crawl finished" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      CrawlFinishedWorker.queue_size.should == 1
+    end
+  end
+  describe "with limited mime_types" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :quiet => true,
+        :cache => nil,
+        :queue_system => :sidekiq,
+        :valid_mime_types => ["text/html"]
+      }
+      @cobweb = Cobweb.new @request
+    end
+      
+    it "should only crawl html pages" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      CrawlProcessWorker.queue_size.should == 8
+
+      mime_types = CrawlProcessWorker.queue_items(0, 100).map{|job| JSON.parse(job)["args"][0]["mime_type"]}
+
+      mime_types.count.should == 8
+      mime_types.map{|m| m.should == "text/html"}
+      mime_types.select{|m| m=="text/html"}.count.should == 8
+      
+      
+    end
+    
+  end
+  describe "with a crawl limit" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :quiet => true,
+        :queue_system => :sidekiq,
+        :cache => nil
+      }
+    end
+  
+    describe "of 1" do
+      before(:each) do
+        @request[:crawl_limit] = 1
+        @cobweb = Cobweb.new @request
+      end
+  
+      it "should not crawl the entire site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should_not == @base_page_count
+      end      
+      it "should only crawl 1 page" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should == 1
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlFinishedWorker.queue_size.should == 1
+      end      
+    
+    end
+  
+    describe "of 5" do
+      before(:each) do
+        @request[:crawl_limit] = 5
+      end
+
+      describe "limiting count to html pages only" do
+        before(:each) do
+          @request[:crawl_limit_by_page] = true
+          @cobweb = Cobweb.new @request
+        end
+      
+        it "should only use html pages towards the crawl limit" do
+          crawl = @cobweb.start(@base_url)
+          @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+          wait_for_crawl_finished crawl[:crawl_id]
+        
+          mime_types = CrawlProcessWorker.queue_items(0, 200).map{|job| JSON.parse(job)["args"][0]["mime_type"]}
+          ap mime_types
+          mime_types.select{|m| m=="text/html"}.count.should == 5
+        end
+      end
+    end
+  
+    describe "of 10" do
+      before(:each) do
+        @request[:crawl_limit] = 10
+        @cobweb = Cobweb.new @request
+      end
+      
+      it "should not crawl the entire site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should_not == @base_page_count
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlFinishedWorker.queue_size.should == 1
+      end      
+      it "should only crawl 10 objects" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should == 10
+      end
+    end
+  
+    describe "of 100" do
+      before(:each) do
+        @request[:crawl_limit] = 100
+        @cobweb = Cobweb.new @request
+      end
+      
+      it "should crawl the entire sample site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should == @base_page_count
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlFinishedWorker.queue_size.should == 1
+      end      
+      it "should not crawl 100 pages" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        CrawlProcessWorker.queue_size.should_not == 100
+      end      
+    end    
+  end
+
+  after(:all) do
+    @all_processes = `ps aux | grep sidekiq | grep -v grep | grep -v sidekiq-web | awk '{print $2}'`.split("\n")
+    unless (@all_processes - @existing_processes).empty?
+      command = "kill #{(@all_processes - @existing_processes).join(" ")}"
+      IO.popen(command)
+    end
+    clear_queues
+  end
+
+end
+
+def wait_for_crawl_finished(crawl_id, timeout=20)
+  @counter = 0
+  start_time = Time.now
+  while(running?(crawl_id) && Time.now < start_time + timeout) do
+    sleep 1
+  end
+  if Time.now > start_time + timeout
+    raise "End of crawl not detected"
+  end
+end
+
+def running?(crawl_id)
+  status = @stat.get_status
+  result = true
+  if status == CobwebCrawlHelper::STARTING
+    result = true
+  else
+    if status == @last_stat
+      if @counter > 20
+        raise "Static status: #{status}"
+      else
+        @counter += 1
+      end
+    else
+      result = status != CobwebCrawlHelper::FINISHED && status != CobwebCrawlHelper::CANCELLED
+    end
+  end
+  @last_stat = @stat.get_status
+  result
+end
+
+def clear_queues
+  Sidekiq.redis do |conn|
+    conn.smembers("queues").each do |queue_name|
+      conn.del("queue:#{queue_name}")
+      conn.srem("queues", queue_name)
+    end
+  end
+  sleep 2
+  
+  CrawlProcessWorker.queue_size.should == 0
+  CrawlFinishedWorker.queue_size.should == 0
+end
+
+

data/spec/cobweb/robots_spec.rb

@@ -1,8 +1,9 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe Robots do
-
+  include HttpStubs
   before(:each) do
+    setup_stubs
     @cobweb = Cobweb.new :quiet => true, :cache => nil
   end
   

data/spec/http_stubs.rb

@@ -0,0 +1,95 @@
+module HttpStubs
+
+  def setup_stubs
+
+    @default_headers = {"Cache-Control" => "private, max-age=0",
+                        "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
+                        "Expires" => "-1",
+                        "Content-Type" => "text/html; charset=UTF-8",
+                        "Content-Encoding" => "",
+                        "Transfer-Encoding" => "chunked",
+                        "Server" => "gws",
+                        "X-XSS-Protection" => "1; mode=block"}
+
+    @symbolized_default_headers = {:"Cache-Control" => "private, max-age=0",
+                        :"Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
+                        :"Expires" => "-1",
+                        :"Content-Type" => "text/html; charset=UTF-8",
+                        :"Content-Encoding" => "",
+                        :"Transfer-Encoding" => "chunked",
+                        :"Server" => "gws",
+                        :"X-XSS-Protection" => "1; mode=block"}
+    
+
+    @mock_http_client = mock(Net::HTTP)
+    @mock_http_request = mock(Net::HTTPRequest)
+    @mock_http_robot_request = mock(Net::HTTPRequest)
+    @mock_http_redirect_request = mock(Net::HTTPRequest)
+    @mock_http_redirect_request2 = mock(Net::HTTPRequest)
+    
+    @mock_http_response = mock(Net::HTTPResponse)
+    @mock_http_robot_response = mock(Net::HTTPResponse)
+    @mock_http_redirect_response = mock(Net::HTTPRedirection)
+    @mock_http_redirect_response2 = mock(Net::HTTPRedirection)
+    @mock_http_get = mock(Net::HTTP::Get)
+    
+    Net::HTTP.stub!(:new).and_return(@mock_http_client)
+    Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub!(:new).with("/robots.txt", an_instance_of(Hash)).and_return(@mock_http_robot_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect2.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request2)
+    Net::HTTP::Get.stub!(:new).with("/redirected.html", an_instance_of(Hash)).and_return(@mock_http_request)
+
+    Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
+
+    @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
+    @mock_http_client.stub!(:request).with(@mock_http_robot_request).and_return(@mock_http_robot_response)
+    @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
+    @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
+    @mock_http_client.stub!(:read_timeout=).and_return(nil)      
+    @mock_http_client.stub!(:open_timeout=).and_return(nil)      
+    @mock_http_client.stub!(:start).and_return(@mock_http_response)
+    @mock_http_client.stub!(:address).and_return("www.baseurl.com")
+    @mock_http_client.stub!(:port).and_return("80")
+    
+    @mock_http_robot_response.stub!(:code).and_return(200)
+    @mock_http_robot_response.stub!(:body).and_return(File.open(File.dirname(__FILE__) + '/../spec/samples/robots.txt', "r").read)
+    @mock_http_robot_response.stub!(:content_type).and_return("text/plain")
+    @mock_http_robot_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_robot_response.stub!(:[]).with("location").and_return(@default_headers["location"])
+    @mock_http_robot_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_robot_response.stub!(:content_length).and_return(1024)
+    @mock_http_robot_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_robot_response.stub!(:to_hash).and_return(@default_headers)
+    
+    @mock_http_response.stub!(:code).and_return(200)
+    @mock_http_response.stub!(:content_type).and_return("text/html")
+    @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_response.stub!(:[]).with("location").and_return(@default_headers["location"])
+    @mock_http_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_response.stub!(:content_length).and_return(1024)
+    @mock_http_response.stub!(:body).and_return("asdf")
+    @mock_http_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_response.stub!(:to_hash).and_return(@default_headers)
+    
+    @mock_http_redirect_response.stub!(:code).and_return(301)
+    @mock_http_redirect_response.stub!(:content_type).and_return("text/html")
+    @mock_http_redirect_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
+    @mock_http_redirect_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response.stub!(:content_length).and_return(2048)
+    @mock_http_redirect_response.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
+    
+    @mock_http_redirect_response2.stub!(:code).and_return(301)
+    @mock_http_redirect_response2.stub!(:content_type).and_return("text/html")
+    @mock_http_redirect_response2.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
+    @mock_http_redirect_response2.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
+    @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
+    @mock_http_redirect_response2.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
+  end
+end

data/spec/samples/sample_site/{boxgrid.html → boxgrid>withsillyname.html}

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li class="current"><a href="boxgrid.html">Boxes Grid</a></li>
+			<li class="current"><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/dashboard.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/forms.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li class="current"><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/gallery.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li class="current"><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/more.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/tables.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li class="current"><a href="tables.html">Tables</a></li>

data/spec/samples/sample_site/typography.html

@@ -71,7 +71,7 @@
 				</ul> 
 			</li> 
 			<li class="current"><a href="typography.html">Typography</a></li>
-			<li><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="boxgrid>withsillyname.html">Boxes Grid</a></li>
 			<li><a href="forms.html">Forms</a></li>
 			<li><a href="gallery.html">Gallery</a></li>
 			<li><a href="tables.html">Tables</a></li>

data/spec/spec_helper.rb

@@ -1,7 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/../lib/sidekiq/cobweb_helper')
 require File.expand_path(File.dirname(__FILE__) + '/../lib/cobweb')
 require File.expand_path(File.dirname(__FILE__) + '/../spec/samples/sample_server')
+require File.expand_path(File.dirname(__FILE__) + '/../spec/http_stubs')
 require 'mock_redis'
 require 'thin' if ENV["TRAVIS_RUBY_VERSION"].nil?
+require 'sidekiq'
+
+require 'coveralls'
+Coveralls.wear!
 
 # Sets up the environment as test so that exceptions are raised
 ENVIRONMENT = "test"
@@ -32,94 +38,6 @@ RSpec.configure do |config|
     
     #redis_mock.flushdb
     
-    @default_headers = {"Cache-Control" => "private, max-age=0",
-                        "Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
-                        "Expires" => "-1",
-                        "Content-Type" => "text/html; charset=UTF-8",
-                        "Content-Encoding" => "",
-                        "Transfer-Encoding" => "chunked",
-                        "Server" => "gws",
-                        "X-XSS-Protection" => "1; mode=block"}
-
-    @symbolized_default_headers = {:"Cache-Control" => "private, max-age=0",
-                        :"Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
-                        :"Expires" => "-1",
-                        :"Content-Type" => "text/html; charset=UTF-8",
-                        :"Content-Encoding" => "",
-                        :"Transfer-Encoding" => "chunked",
-                        :"Server" => "gws",
-                        :"X-XSS-Protection" => "1; mode=block"}
-
-    @mock_http_client = mock(Net::HTTP)
-    @mock_http_request = mock(Net::HTTPRequest)
-    @mock_http_robot_request = mock(Net::HTTPRequest)
-    @mock_http_redirect_request = mock(Net::HTTPRequest)
-    @mock_http_redirect_request2 = mock(Net::HTTPRequest)
-    
-    @mock_http_response = mock(Net::HTTPResponse)
-    @mock_http_robot_response = mock(Net::HTTPResponse)
-    @mock_http_redirect_response = mock(Net::HTTPRedirection)
-    @mock_http_redirect_response2 = mock(Net::HTTPRedirection)
-    @mock_http_get = mock(Net::HTTP::Get)
-    
-    Net::HTTP.stub!(:new).and_return(@mock_http_client)
-    Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request)
-    Net::HTTP::Get.stub!(:new).with("/robots.txt", an_instance_of(Hash)).and_return(@mock_http_robot_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect2.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request2)
-    Net::HTTP::Get.stub!(:new).with("/redirected.html", an_instance_of(Hash)).and_return(@mock_http_request)
-
-    Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
-
-    @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
-    @mock_http_client.stub!(:request).with(@mock_http_robot_request).and_return(@mock_http_robot_response)
-    @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
-    @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
-    @mock_http_client.stub!(:read_timeout=).and_return(nil)      
-    @mock_http_client.stub!(:open_timeout=).and_return(nil)      
-    @mock_http_client.stub!(:start).and_return(@mock_http_response)
-    @mock_http_client.stub!(:address).and_return("www.baseurl.com")
-    @mock_http_client.stub!(:port).and_return("80")
-    
-    @mock_http_robot_response.stub!(:code).and_return(200)
-    @mock_http_robot_response.stub!(:body).and_return(File.open(File.dirname(__FILE__) + '/../spec/samples/robots.txt', "r").read)
-    @mock_http_robot_response.stub!(:content_type).and_return("text/plain")
-    @mock_http_robot_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_robot_response.stub!(:[]).with("location").and_return(@default_headers["location"])
-    @mock_http_robot_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_robot_response.stub!(:content_length).and_return(1024)
-    @mock_http_robot_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_robot_response.stub!(:to_hash).and_return(@default_headers)
-    
-    @mock_http_response.stub!(:code).and_return(200)
-    @mock_http_response.stub!(:content_type).and_return("text/html")
-    @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_response.stub!(:[]).with("location").and_return(@default_headers["location"])
-    @mock_http_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_response.stub!(:content_length).and_return(1024)
-    @mock_http_response.stub!(:body).and_return("asdf")
-    @mock_http_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_response.stub!(:to_hash).and_return(@default_headers)
-    
-    @mock_http_redirect_response.stub!(:code).and_return(301)
-    @mock_http_redirect_response.stub!(:content_type).and_return("text/html")
-    @mock_http_redirect_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
-    @mock_http_redirect_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_redirect_response.stub!(:content_length).and_return(2048)
-    @mock_http_redirect_response.stub!(:body).and_return("redirected body")
-    @mock_http_redirect_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
-    
-    @mock_http_redirect_response2.stub!(:code).and_return(301)
-    @mock_http_redirect_response2.stub!(:content_type).and_return("text/html")
-    @mock_http_redirect_response2.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
-    @mock_http_redirect_response2.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
-    @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
-    @mock_http_redirect_response2.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
   }
 
 end