cobweb 0.0.1 → 0.0.2

data/README.textile

@@ -1,9 +1,9 @@
 
-h1. Cobweb v0.0.1
+h1. Cobweb v0.0.2
 
 h2. Intro 
 
-Crawler that utilises resque jobs to perform the crawl allowing clustering of crawls. 
+  Crawler that utilises resque jobs to perform the crawl allowing clustering of crawls and redis to perform caching of responses.  It currently requires redis on the same machine but will add options to specify redis server shortly. 
 
 h2. Installation
 
@@ -19,24 +19,31 @@ Creates a new crawler object based on a base_url
 
   * options - Options are passed in as a hash,
   
-    ** :follow_redirects - transparently follows redirects and populates the :redirect_through key in the content hash(true)
-    ** :redirect_limit   - sets the limit to be used for concurrent redirects(10) 
-    ** :processing_queue - specifies the processing queue for content to be sent to (ContentProcessJob) 
-    ** :debug            - enables debug output (false)
-    ** :quiet            - hides default output (false)
-    ** :cache            - if set, enables the cache and sets the ttl (300)
+    ** :follow_redirects - transparently follows redirects and populates the :redirect_through key in the content hash (Default: true)
+    ** :redirect_limit   - sets the limit to be used for concurrent redirects (Default: 10) 
+    ** :processing_queue - specifies the processing queue for content to be sent to (Default: ContentProcessJob) 
+    ** :debug            - enables debug output (Default: false)
+    ** :quiet            - hides default output (Default: false)
+    ** :cache            - sets the ttl for caching pages, set to nil to disable caching (Default: 300)
     
 bq. crawler = CobWeb.new(:follow_redirects => false)
 
-
 h4. start(base_url)
 
+Starts a crawl through resque.  Requires the :processing_queue to be set to a valid class for the resque job to work with the data retrieved.
+
   * base_url - the url to start the crawl from
 
+bq. crawler.start("http://www.google.com/")
+
 h4. get(url)
 
+Simple get that obey's the options supplied in new.
+
   * url - url requested
 
+bq. crawler.get("http://www.google.com/")
+
 
 h2. License
 

data/lib/cobweb.rb

@@ -11,10 +11,11 @@ class CobWeb
   
   def initialize(options = {})
     @options = options
-    @options[:follow_redirects] = true if @options[:follow_redirects].nil?
-    @options[:redirect_limit] = 10 if @options[:redirect_limit].nil? 
-    @options[:processing_queue] = ContentProcessJob if @options[:processing_queue].nil?
-    @options[:debug] = false unless @options[:debug]
+    @options[:follow_redirects] = true unless @options.has_key?(:follow_redirects)
+    @options[:redirect_limit] = 10 unless @options.has_key?(:redirect_limit)
+    @options[:processing_queue] = ContentProcessJob unless @options.has_key?(:processing_queue)
+    @options[:debug] = false unless @options.has_key?(:debug)
+    @options[:cache] = 300 unless @options.has_key?(:cache)
     
   end
   
@@ -30,7 +31,6 @@ class CobWeb
     Resque.enqueue(CrawlJob, request)
   end
 
-
   def get(url, redirect_limit = @options[:redirect_limit])
 
     raise "url cannot be nil" if url.nil?    
@@ -90,7 +90,6 @@ class CobWeb
         content[:content_body] = response.body
         content[:location] = response["location"]
         content[:headers] = response.to_hash.symbolize_keys
-        
         # parse data for links
         link_parser = ContentLinkParser.new(content[:url], content[:content_body])
         content[:links] = link_parser.link_data
@@ -98,7 +97,7 @@ class CobWeb
         # add content to cache if required
         if @options[:cache]
           redis.set(unique_id, content.to_json)
-          redis.expire unique_id, content_request[:cache].to_i
+          redis.expire unique_id, @options[:cache].to_i
         end
       end
     end

data/spec/cobweb/cobweb_spec.rb

@@ -15,7 +15,7 @@ describe CobWeb do
                         "Server" => "gws",
                         "X-XSS-Protection" => "1; mode=block"}
 
-    @cobweb = CobWeb.new :quiet => true
+    @cobweb = CobWeb.new :quiet => true, :cache => nil
   end  
 
   describe "with mock" do
@@ -24,11 +24,11 @@ describe CobWeb do
       @mock_http_request = mock(Net::HTTPRequest)
       @mock_http_redirect_request = mock(Net::HTTPRequest)
       @mock_http_redirect_request2 = mock(Net::HTTPRequest)
-
+      
       @mock_http_response = mock(Net::HTTPResponse)
       @mock_http_redirect_response = mock(Net::HTTPRedirection)
       @mock_http_get = mock(Net::HTTP::Get)
-  
+      
       Net::HTTP.stub!(:new).and_return(@mock_http_client)
       Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
       Net::HTTP::Get.stub!(:new).with("/redirect.html").and_return(@mock_http_redirect_request)
@@ -37,7 +37,7 @@ describe CobWeb do
       @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
       @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
       @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)      
-  
+      
       @mock_http_response.stub!(:code).and_return(200)
       @mock_http_response.stub!(:content_type).and_return("text/html")
       @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
@@ -72,36 +72,31 @@ describe CobWeb do
       it "should return a hash with default values" do
         @cobweb.get(@base_url).should be_an_instance_of Hash
       end
-  
+      
       it "should return a hash with default values without quiet option" do
         @cobweb.get(@base_url).should be_an_instance_of Hash
       end
-    
+      
       it "should raise exception if there is no url" do
         lambda {@cobweb.get(nil)}.should raise_error("url cannot be nil")
       end
-  
-      describe "content object" do
       
+      describe "content object" do
         it "should return the url" do
           @cobweb.get(@base_url)[:url].should == @base_url
         end
-        
-        it "should return correct content-types" do
+        it "should return correct content-type" do
           @mock_http_response.stub!(:content_type).and_return("image/jpeg")
           @cobweb.get(@base_url)[:content_type].should == "image/jpeg"
         end
-        
         it "should return correct status-code" do
           @mock_http_response.stub!(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
         end
-        
         it "should return correct status-code" do
           @mock_http_response.stub!(:code).and_return(404)
           @cobweb.get(@base_url)[:status_code].should == 404
         end
-        
         it "should return correct character_set" do
           @cobweb.get(@base_url)[:character_set].should == "UTF-8"
         end 
@@ -126,19 +121,18 @@ describe CobWeb do
         it "should return the response time for the url" do
           @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
         end
-        
       end
       describe "with redirect" do
         
         before(:each) do
           @base_url = "http://redirect-me.com/redirect.html"
-          @cobweb = CobWeb.new(:follow_redirects => true, :quiet => true)
+          @cobweb = CobWeb.new(:follow_redirects => true, :quiet => true, :cache => nil)
         end
         
         it "should flow through redirect" do
-
+          
           @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)        
-        
+          
           content = @cobweb.get(@base_url)
           content.should be_an_instance_of Hash
           
@@ -150,15 +144,15 @@ describe CobWeb do
         end
         it "should return the path followed" do
           @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)        
-        
+          
           content = @cobweb.get(@base_url)
           content[:redirect_through].should == ["http://redirected-to.com/redirect2.html", "http://redirected-to.com/redirected.html"]          
           
         end
         it "should not follow with redirect disabled" do
-          @cobweb = CobWeb.new(:follow_redirects => false)
+          @cobweb = CobWeb.new(:follow_redirects => false, :cache => nil)
           @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)        
-        
+          
           content = @cobweb.get(@base_url)
           content[:url].should == "http://redirect-me.com/redirect.html"
           content[:redirect_through].should be_nil

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cobweb
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Stewart McKee
@@ -88,7 +88,6 @@ files:
 - spec/spec_helper.rb
 - spec/cobweb/content_link_parser_spec.rb
 - spec/cobweb/cobweb_spec.rb
-- spec/cobweb/crawl_job_spec.rb
 - lib/namespaced_redis.rb
 - lib/cobweb.rb
 - lib/content_process_job.rb

data/spec/cobweb/crawl_job_spec.rb

@@ -1,24 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
-
-
-describe CrawlJob do
-
-  before(:each) do
-    @base_url = "http://www.baseurl.com/"
-
-    
-
-    client = Net::HTTPClient.new
-    puts client.get('http://www.google.com.au')
-    puts "asdf"
-    
-    @cobweb = CobWeb.new("http://www.google.com")
-    
-  end
-  
-  it "should be a cobweb type" do
-    @cobweb.should be_an_instance_of CobWeb
-  end
-  
-
-end