cobweb 0.0.76 → 0.0.77

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.76
+h1. Cobweb v0.0.77
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.76"
+    "0.0.77"
   end
 
 end

data/lib/crawl.rb

@@ -50,37 +50,35 @@ module CobwebModule
     end
 
     def retrieve
-      lock("retrieve") do
-        unless @redis.sismember("currently_running", @options[:url])
-          @redis.sadd("currently_running", @options[:url])
-          unless already_crawled?
-            if within_crawl_limits?
-              @stats.update_status("Retrieving #{@options[:url]}...")
-              @content = Cobweb.new(@options).get(@options[:url], @options)
-              if @options[:url] == @redis.get("original_base_url")
-                @redis.set("crawled_base_url", @content[:base_url])
-              end
-              update_queues
+      unless @redis.sismember("currently_running", @options[:url])
+        @redis.sadd("currently_running", @options[:url])
+        unless already_crawled?
+          if within_crawl_limits?
+            @stats.update_status("Retrieving #{@options[:url]}...")
+            @content = Cobweb.new(@options).get(@options[:url], @options)
+            if @options[:url] == @redis.get("original_base_url")
+              @redis.set("crawled_base_url", @content[:base_url])
+            end
+            update_queues
 
-              if content.permitted_type?
-                ## update statistics
+            if content.permitted_type?
+              ## update statistics
 
-                @stats.update_statistics(@content)
-                return true
-              end
-            else
-              decrement_queue_counter
+              @stats.update_statistics(@content)
+              return true
             end
           else
             decrement_queue_counter
           end
         else
-          debug_puts "\n\nDETECTED DUPLICATE JOB for #{@options[:url]}\n"
-          debug_ap @redis.smembers("currently_running")
           decrement_queue_counter
         end
-        false
+      else
+        debug_puts "\n\nDETECTED DUPLICATE JOB for #{@options[:url]}\n"
+        debug_ap @redis.smembers("currently_running")
+        decrement_queue_counter
       end
+      false
     end
 
     def process_links &block
@@ -156,7 +154,7 @@ module CobwebModule
         increment_process_counter
       end
       @redis.sadd "enqueued", @options[:url]
-      
+
       yield if block_given?
       @redis.incr("crawl_job_enqueued_count")
     end
@@ -181,7 +179,7 @@ module CobwebModule
     end
 
     def finished
-      set_first_to_finish 
+      set_first_to_finish
       debug_ap "CRAWL FINISHED  #{@options[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}"
       @stats.end_crawl(@options)
     end
@@ -204,7 +202,7 @@ module CobwebModule
     def first_to_finish?
       @first_to_finish
     end
-    
+
     def crawled_base_url
       @redis.get("crawled_base_url")
     end
@@ -228,6 +226,7 @@ module CobwebModule
       end
 
       debug_puts "RECEIVED LOCK [#{key}]"
+      @redis.expire("#{key}_lock", 10)
       begin
         result = yield
       ensure
@@ -236,15 +235,15 @@ module CobwebModule
       end
       result
     end
-    
+
     def debug_ap(value)
       ap(value) if @options[:debug]
     end
-    
+
     def debug_puts(value)
       puts(value) if @options[:debug]
     end
-    
+
     private
     def setup_defaults
       @options[:redis_options] = {} unless @options.has_key? :redis_options

data/lib/crawl_job.rb

@@ -29,30 +29,28 @@ class CrawlJob
 
         end
         
-        @crawl.lock("crawl_job_process") do
-          if @crawl.to_be_processed?
-            
-            @crawl.process do
+        if @crawl.to_be_processed?
+          
+          @crawl.process do
 
-              # enqueue to processing queue
-              @crawl.debug_puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
-              send_to_processing_queue(@crawl.content.to_hash, content_request)
+            # enqueue to processing queue
+            @crawl.debug_puts "ENQUEUED [#{@crawl.redis.get("crawl_job_enqueued_count")}] URL: #{@crawl.content.url}"
+            send_to_processing_queue(@crawl.content.to_hash, content_request)
 
-              #if the enqueue counter has been requested update that
-              if content_request.has_key?(:enqueue_counter_key)
-                enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
-                current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
-                enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
-              end
-              
+            #if the enqueue counter has been requested update that
+            if content_request.has_key?(:enqueue_counter_key)
+              enqueue_redis = Redis::Namespace.new(content_request[:enqueue_counter_namespace].to_s, :redis => Redis.new(content_request[:redis_options]))
+              current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+              enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
             end
-          else
-            @crawl.debug_puts "@crawl.finished? #{@crawl.finished?}"
-            @crawl.debug_puts "@crawl.within_crawl_limits? #{@crawl.within_crawl_limits?}"
-            @crawl.debug_puts "@crawl.first_to_finish? #{@crawl.first_to_finish?}"
+            
           end
-          
+        else
+          @crawl.debug_puts "@crawl.finished? #{@crawl.finished?}"
+          @crawl.debug_puts "@crawl.within_crawl_limits? #{@crawl.within_crawl_limits?}"
+          @crawl.debug_puts "@crawl.first_to_finish? #{@crawl.first_to_finish?}"
         end
+        
       end
     end
     

data/spec/cobweb/cobweb_job_spec.rb

@@ -262,16 +262,10 @@ def running?(crawl_id)
   else
     if status == @last_stat
       if @counter > 20
-        puts ""
         raise "Static status: #{status}"
       else
         @counter += 1
       end
-      if @counter == 1
-        print "Static Status.. #{21-@counter}"
-      else
-        print ".#{21-@counter}"
-      end
     else
       result = status != CobwebCrawlHelper::FINISHED && status != CobwebCrawlHelper::CANCELLED
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.76
+  version: 0.0.77
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-18 00:00:00.000000000 Z
+date: 2012-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70301036171860 !ruby/object:Gem::Requirement
+  requirement: &70355025183680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036171860
+  version_requirements: *70355025183680
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70301036170940 !ruby/object:Gem::Requirement
+  requirement: &70355025182000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036170940
+  version_requirements: *70355025182000
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70301036169780 !ruby/object:Gem::Requirement
+  requirement: &70355025180660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036169780
+  version_requirements: *70355025180660
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70301036169040 !ruby/object:Gem::Requirement
+  requirement: &70355025179740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036169040
+  version_requirements: *70355025179740
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70301036167760 !ruby/object:Gem::Requirement
+  requirement: &70355025179160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036167760
+  version_requirements: *70355025179160
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70301036166240 !ruby/object:Gem::Requirement
+  requirement: &70355025178560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036166240
+  version_requirements: *70355025178560
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70301036164760 !ruby/object:Gem::Requirement
+  requirement: &70355025177760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036164760
+  version_requirements: *70355025177760
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70301036163660 !ruby/object:Gem::Requirement
+  requirement: &70355025177180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036163660
+  version_requirements: *70355025177180
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70301036162760 !ruby/object:Gem::Requirement
+  requirement: &70355025176740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036162760
+  version_requirements: *70355025176740
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70301036161620 !ruby/object:Gem::Requirement
+  requirement: &70355025175980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70301036161620
+  version_requirements: *70355025175980
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70301036161100 !ruby/object:Gem::Requirement
+  requirement: &70355025175280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,9 +131,9 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301036161100
+  version_requirements: *70355025175280
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
-  crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
+  crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface
   to monitor the progress of the crawls.
 email: stewart@rockwellcottage.com