cobweb 1.0.27 → 1.0.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cee480edd847679803dcc5e7b47f03523bd5775
-  data.tar.gz: 37ebf10e098d2b46274a80a969962eadea9bd307
+  metadata.gz: 4ee0943ca3fabf5cb097b9d6edb324783cc155d6
+  data.tar.gz: d79eca877414244d94d5e66062a368bee4104368
 SHA512:
-  metadata.gz: ea3ae531762c16268aac13babd9dbdb482bed8930819de4cdb013f268378eb729483512f181f85ad421a40b19ae2086011d46e6f459286cb314889450be7329c
-  data.tar.gz: ca1bf5d58f6a242af1030fd2d5f83c28444f8eae47e12cb1c6938032687559824924ef313b52e276186c2c480137035258db5154fcf0205445a7b4c811644a14
+  metadata.gz: 420d1fe0daff99694de78846491f97357d42632a78cdc9a29953f95afe93223c40ecbf2fc09b34bf378321322ff9a7958f28a45b352e629d4750af56cb192a0f
+  data.tar.gz: e9e38c6a2c33ffff09e76c04e71f673512523a417b37b001a87142ffd8a01148b40140d1c98fb263b562e2278c0ebdb08e830a65467c41ad3fdb2ef654938012

data/README.textile

@@ -1,4 +1,4 @@
-h1. Cobweb v1.0.27
+h1. Cobweb v1.0.28
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb

data/lib/cobweb_crawl_helper.rb

@@ -30,25 +30,6 @@ class CobwebCrawlHelper
     # set status as cancelled now so that we don't enqueue any further pages
     self.statistics.end_crawl(@data, true)
 
-
-    if options[:crawl_finished_queue] && options[:queue_system] == :resque && RESQUE_INSTALLED
-
-      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
-      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
-      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
-
-      Resque.enqueue(options[:crawl_finished_queue], @stats.get_statistics.merge(additional_stats))
-    end
-
-    if options[:crawl_finished_queue] && options[:queue_system] == :sidekiq && SIDEKIQ_INSTALLED
-
-      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
-      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
-      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
-
-      Kernel.const_get(options[:crawl_finished_queue]).perform_async(@stats.get_statistics.merge(additional_stats))
-    end
-
     counter = 0
     while(counter < 200) do
       break if self.statistics.get_status == CANCELLED
@@ -70,14 +51,43 @@ class CobwebCrawlHelper
       end
     end
     if options[:queue_system] == :sidekiq && SIDEKIQ_INSTALLED
-      queue_name = Kernel.const_get(options[:processing_queue]).sidekiq_options_hash["queue"]
-      queue = Sidekiq::Queue.new(queue_name)
+
+      puts "deleteing from crawl_worker"
+      queue = Sidekiq::Queue.new("crawl_worker")
+      queue.each do |job|
+        ap job.args # => [1, 2, 3]
+        job.delete if job.args[0]["crawl_id"] == id
+      end
+
+
+      process_queue_name = Kernel.const_get(options[:processing_queue]).sidekiq_options_hash["queue"]
+      puts "deleting from #{process_queue_name}"
+      queue = Sidekiq::Queue.new(process_queue_name)
       queue.each do |job|
-        job.args # => [1, 2, 3]
+        ap job.args # => [1, 2, 3]
         job.delete if job.args[0]["crawl_id"] == id
       end
     end
 
+    if options[:crawl_finished_queue] && options[:queue_system] == :resque && RESQUE_INSTALLED
+
+      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
+      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
+
+      Resque.enqueue(options[:crawl_finished_queue], @stats.get_statistics.merge(additional_stats))
+    end
+
+    if options[:crawl_finished_queue] && options[:queue_system] == :sidekiq && SIDEKIQ_INSTALLED
+
+      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
+      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
+
+      Kernel.const_get(options[:crawl_finished_queue]).perform_async(@stats.get_statistics.merge(additional_stats))
+    end
+
+
   end
 
   def statistics

data/lib/cobweb_crawler.rb

@@ -1,6 +1,5 @@
 require 'digest/md5'
 require 'date'
-require 'ap'
 require 'redis-namespace'
 
 # CobwebCrawler is a standalone crawler, it includes a built in statistics monitor using Sinatra.

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
 
   # Returns a string of the current version
   def self.version
-    "1.0.27"
+    "1.0.28"
   end
 
 end

data/lib/crawl.rb

@@ -30,6 +30,10 @@ module CobwebModule
       already_crawled?(link) || already_queued?(link) || already_running?(link)
     end
 
+    def cancelled?
+      @stats.get_statistics[:current_status] == "Cancelled"
+    end
+
     # Returns true if the crawl count is within limits
     def within_crawl_limits?
       @options[:crawl_limit].nil? || crawl_counter < @options[:crawl_limit].to_i
@@ -58,42 +62,47 @@ module CobwebModule
     end
 
     def retrieve
+      unless cancelled?
+        unless already_running? @options[:url]
+          unless already_crawled? @options[:url]
+            update_queues
+            if within_crawl_limits?
+              @redis.sadd("currently_running", @options[:url])
+              @stats.update_status("Retrieving #{@options[:url]}...")
+              @content = Cobweb.new(@options).get(@options[:url], @options)
+              update_counters
+
+              if @options[:url] == @redis.get("original_base_url")
+                @redis.set("crawled_base_url", @content[:base_url])
+              end
 
-      unless already_running? @options[:url]
-        unless already_crawled? @options[:url]
-          update_queues
-          if within_crawl_limits?
-            @redis.sadd("currently_running", @options[:url])
-            @stats.update_status("Retrieving #{@options[:url]}...")
-            @content = Cobweb.new(@options).get(@options[:url], @options)
-            update_counters
-
-            if @options[:url] == @redis.get("original_base_url")
-              @redis.set("crawled_base_url", @content[:base_url])
-            end
-
-            if content.permitted_type?
-              ## update statistics
+              if content.permitted_type?
+                ## update statistics
 
-              @stats.update_statistics(@content)
-              return true
+                @stats.update_statistics(@content)
+                return true
+              end
+            else
+              puts "======================================="
+              puts "OUTWITH CRAWL LIMITS"
+              puts "======================================="
+              decrement_queue_counter
             end
           else
             puts "======================================="
-            puts "OUTWITH CRAWL LIMITS"
+            puts "ALREADY CRAWLED"
             puts "======================================="
             decrement_queue_counter
           end
         else
-          puts "======================================="
-          puts "ALREADY CRAWLED"
-          puts "======================================="
+          debug_puts "\n\nDETECTED DUPLICATE JOB for #{@options[:url]}\n"
+          debug_ap @redis.smembers("currently_running")
           decrement_queue_counter
         end
       else
-        debug_puts "\n\nDETECTED DUPLICATE JOB for #{@options[:url]}\n"
-        debug_ap @redis.smembers("currently_running")
-        decrement_queue_counter
+        puts "======================================="
+        puts "CRAWL CANCELLED"
+        puts "======================================="
       end
       false
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.27
+  version: 1.0.28
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-18 00:00:00.000000000 Z
+date: 2015-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis