cobweb 0.0.71 → 0.0.72

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.71
+h1. Cobweb v0.0.72
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb_crawl_helper.rb

@@ -10,28 +10,41 @@ class CobwebCrawlHelper
   
   def initialize(data)
     @data = data
+    
+    # TAKING A LONG TIME TO RUN ON PRODUCTION BOX
     @stats = Stats.new(data)
   end
   
-  def destroy
-    queue_name = "cobweb_crawl_job"
+  def destroy(options)
+    
+    options[:queue_name] = "cobweb_crawl_job" unless options.has_key?(:queue_name)
+    options[:finished_resque_queue] = CobwebFinishedJob unless options.has_key?(:finished_resque_queue)
+    
     # set status as cancelled now so that we don't enqueue any further pages
     self.statistics.end_crawl(@data, true)
+    puts "end_crawl: #{self.statistics.get_status}"
+    if options[:finished_resque_queue]
+      puts "enqueueing finished job..."
+      
+      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
+      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
+      
+      Resque.enqueue(options[:finished_resque_queue], @stats.get_statistics.merge(additional_stats))
+    end
     
-    job_items = Resque.peek(queue_name, 0, BATCH_SIZE)
-    batch_count = 0
+    position = 0
+    job_items = Resque.peek(options[:queue_name], position, BATCH_SIZE)
     until job_items.empty?
-      
+      puts "Batch: #{position} : #{job_items.count}"
       job_items.each do |item|
         if item["args"][0]["crawl_id"] == id
-          # remote this job from the queue
+          # remove this job from the queue
           Resque.dequeue(CrawlJob, item["args"][0])
         end
       end
       
-      position = batch_count*BATCH_SIZE
-      batch_count += 1
-      job_items = Resque.peek(queue_name, position, BATCH_SIZE)
+      job_items = Resque.peek(options[:queue_name], position+=BATCH_SIZE, BATCH_SIZE)
     end
     
   end

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.71"
+    "0.0.72"
   end
 
 end

data/lib/stats.rb

@@ -1,6 +1,9 @@
 # Stats class is the main statisitics hub for monitoring crawls.  Either can be viewed through the Sinatra interface, or returned from the CobwebCrawler.crawl method or block
 class Stats
   require 'json'
+  
+  attr_reader :redis
+  
   # Sets up redis usage for statistics
   def initialize(options)
     options[:redis_options] = {} unless options.has_key? :redis_options
@@ -131,7 +134,7 @@ class Stats
     @statistics[:status_counts] = status_counts.to_json
     
     ## time based statistics
-    increment_time_stat("minute_totals", "minute", 60)    
+    increment_time_stat("minute_totals", "minute", 60)
     
     redis_command = "@redis.hmset 'statistics', #{@statistics.keys.map{|key| "'#{key}', '#{@statistics[key].to_s.gsub("'","''")}'"}.join(", ")}"
     instance_eval redis_command
@@ -158,7 +161,7 @@ class Stats
   
   # Sets the current status of the crawl
   def update_status(status)
-    #@redis.hset("statistics", "current_status", status) unless status == CobwebCrawlHelper::CANCELLED
+    @redis.hset("statistics", "current_status", status) unless get_status == CobwebCrawlHelper::CANCELLED
   end
   
   # Returns the current status of the crawl

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.71
+  version: 0.0.72
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-16 00:00:00.000000000 Z
+date: 2012-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70207650005840 !ruby/object:Gem::Requirement
+  requirement: &70308627743380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650005840
+  version_requirements: *70308627743380
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70207650005340 !ruby/object:Gem::Requirement
+  requirement: &70308627741840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650005340
+  version_requirements: *70308627741840
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70207650004420 !ruby/object:Gem::Requirement
+  requirement: &70308627741060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650004420
+  version_requirements: *70308627741060
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70207650003640 !ruby/object:Gem::Requirement
+  requirement: &70308627740200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650003640
+  version_requirements: *70308627740200
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70207650002440 !ruby/object:Gem::Requirement
+  requirement: &70308627739500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650002440
+  version_requirements: *70308627739500
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70207650001860 !ruby/object:Gem::Requirement
+  requirement: &70308627739020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650001860
+  version_requirements: *70308627739020
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70207650001420 !ruby/object:Gem::Requirement
+  requirement: &70308627738100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650001420
+  version_requirements: *70308627738100
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70207650000420 !ruby/object:Gem::Requirement
+  requirement: &70308627737580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207650000420
+  version_requirements: *70308627737580
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70207649999520 !ruby/object:Gem::Requirement
+  requirement: &70308627737040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207649999520
+  version_requirements: *70308627737040
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70207649998760 !ruby/object:Gem::Requirement
+  requirement: &70308627736400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70207649998760
+  version_requirements: *70308627736400
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70207649998280 !ruby/object:Gem::Requirement
+  requirement: &70308627735860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70207649998280
+  version_requirements: *70308627735860
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface