cobweb 0.0.37 → 0.0.38

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.37
+h1. Cobweb v0.0.38
 
 h2. Intro 
   
@@ -98,6 +98,8 @@ bq. crawler.head("http://www.google.com/")
 h3. Contributing/Testing
 
   Feel free to contribute small or large bits of code, just please make sure that there are rspec test for the features your submitting.  We also test on travis at http://travis-ci.org/#!/stewartmckee/cobweb if you want to see the state of the project.
+  
+    Continuous integration testing is performed by the excellent Travis: http://travis-ci.org/#!/stewartmckee/cobweb
 
 h2. License
 

data/lib/cobweb.rb

@@ -20,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.37"
+    "0.0.38"
   end
   
   def method_missing(method_sym, *arguments, &block)

data/lib/crawl_job.rb

@@ -10,8 +10,9 @@ class CrawlJob
   def self.perform(content_request)
     
     # change all hash keys to symbols
-    content_request = content_request.deep_symbolize_keys
+    content_request = self.deep_symbolize_keys(content_request)
     
+    content_request[:redis_options] = {} unless content_request.has_key? :redis_options
     @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
     
     @debug = content_request[:debug]
@@ -62,18 +63,12 @@ class CrawlJob
         end
 
         # if there's nothing left queued or the crawled limit has been reached
-        if @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
-
-          # finished
-          stats = @redis.hgetall "statistics"
-          stats[:total_pages] = @redis.get("total_pages").to_i
-          stats[:total_assets] = @redis.get("total_assets").to_i
-          stats[:crawl_counter] = @crawl_counter
-          stats[:queue_counter] = @queue_counter
-          stats[:crawled] = @redis.smembers "crawled"
-
-          Resque.enqueue(const_get(content_request[:crawl_finished_queue]), stats.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))            
-
+        if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
+          if @redis.scard("queued") == 0
+            finished(content_request)
+          end
+        elsif @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+          finished(content_request)
         end
       end
     else
@@ -84,6 +79,14 @@ class CrawlJob
 
   end
 
+  def self.finished(content_request)
+    # finished
+    
+    Stats.set_totals
+
+    Resque.enqueue(const_get(content_request[:crawl_finished_queue]), Stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))                
+  end
+
   def self.send_to_processing_queue(content, content_request)
     content_to_send = content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
     if content_request[:use_encoding_safe_process_job]
@@ -175,4 +178,15 @@ class CrawlJob
     @crawl_counter = @redis.get("crawl-counter").to_i
     @queue_counter = @redis.get("queue-counter").to_i
   end
+  def self.deep_symbolize_keys(hash)
+    hash.keys.each do |key|
+      value = hash[key]
+      hash.delete(key)
+      hash[key.to_sym] = value
+      if hash[key.to_sym].instance_of? Hash
+        hash[key.to_sym] = self.deep_symbolize_keys(hash[key.to_sym])
+      end
+    end
+    hash
+  end
 end

data/lib/stats.rb

@@ -5,13 +5,32 @@ class Stats < Sinatra::Base
   
   def self.update_statistics(statistics)
     @@statistics = statistics
+    @@statistics
+  end
+  
+  def self.get_statistics
+    @@statistics ||= {}
   end
   
   def self.update_status(status)
     @@status = status
   end
   
+  def self.set_totals
+    stats = @redis.hgetall "statistics"
+    stats[:total_pages] = @redis.get("total_pages").to_i
+    stats[:total_assets] = @redis.get("total_assets").to_i
+    stats[:crawl_counter] = @crawl_counter
+    stats[:queue_counter] = @queue_counter
+    stats[:crawled] = @redis.smembers "crawled"
+
+    Stats.update_statistics(stats)
+  end
+  
   def self.set_statistics_in_redis(redis, content)
+    
+    @redis = redis
+    
     crawl_counter = redis.get("crawl-counter").to_i
     queue_counter = redis.get("queue-counter").to_i
     
@@ -62,6 +81,7 @@ class Stats < Sinatra::Base
     end
     redis.hset "statistics", "status_counts", status_counts.to_json
     
+    @@statistics = @redis.hgetall "statistics"
   end
   
   set :views, settings.root + '/../views'

data/spec/cobweb/cobweb_job_spec.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Cobweb do
+
+  before(:each) do
+    @base_url = "http://www.baseurl.com/"
+    @cobweb = Cobweb.new :quiet => true, :cache => nil
+  end
+
+  describe "detect finish of crawl" do
+    
+    describe "with no crawl limit" do
+      before(:each) do
+        @request = {
+          :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+          :url => @base_url,
+          :crawl_limit => nil
+        }
+      end
+      
+      it "should not limit crawl"
+      it "should detect the end or crawl"
+    end
+    
+    describe "with a crawl limit" do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :url => @base_url,
+        :crawl_limit => 3
+      }
+
+      it "should limit crawl"
+      it "should detect the end or crawl based on limit"
+
+    end
+    
+    
+  end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.37
+  version: 0.0.38
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-24 00:00:00.000000000 Z
+date: 2012-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70275220162200 !ruby/object:Gem::Requirement
+  requirement: &70264627738060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220162200
+  version_requirements: *70264627738060
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70275220161740 !ruby/object:Gem::Requirement
+  requirement: &70264627737640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220161740
+  version_requirements: *70264627737640
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70275220177640 !ruby/object:Gem::Requirement
+  requirement: &70264627737220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220177640
+  version_requirements: *70264627737220
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70275220177220 !ruby/object:Gem::Requirement
+  requirement: &70264627736800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220177220
+  version_requirements: *70264627736800
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70275220176640 !ruby/object:Gem::Requirement
+  requirement: &70264627736380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220176640
+  version_requirements: *70264627736380
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70275220176220 !ruby/object:Gem::Requirement
+  requirement: &70264627735960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220176220
+  version_requirements: *70264627735960
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70275220175780 !ruby/object:Gem::Requirement
+  requirement: &70264627735540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220175780
+  version_requirements: *70264627735540
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70275220175360 !ruby/object:Gem::Requirement
+  requirement: &70264627735120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220175360
+  version_requirements: *70264627735120
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70275220174940 !ruby/object:Gem::Requirement
+  requirement: &70264627734700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220174940
+  version_requirements: *70264627734700
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70275220174520 !ruby/object:Gem::Requirement
+  requirement: &70264627734280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70275220174520
+  version_requirements: *70264627734280
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com
@@ -130,6 +130,7 @@ extra_rdoc_files:
 - README.textile
 files:
 - spec/cobweb/cobweb_crawler_spec.rb
+- spec/cobweb/cobweb_job_spec.rb
 - spec/cobweb/cobweb_spec.rb
 - spec/cobweb/content_link_parser_spec.rb
 - spec/samples/sample_html_links.html