cobweb 1.0.11 → 1.0.12

data/README.textile

@@ -1,10 +1,11 @@
 
-h1. Cobweb v1.0.11
+h1. Cobweb v1.0.12
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
-
-!https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
+!https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb
 !https://gemnasium.com/stewartmckee/cobweb.png!
+!https://coveralls.io/repos/stewartmckee/cobweb/badge.png?branch=master(Coverage Status)!:https://coveralls.io/r/stewartmckee/cobweb
+
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -33,8 +33,14 @@ class Cobweb
     default_use_encoding_safe_process_job_to  false
     default_follow_redirects_to               true
     default_redirect_limit_to                 10
-    default_processing_queue_to               "CobwebProcessJob"
-    default_crawl_finished_queue_to           "CobwebFinishedJob"
+    default_queue_system_to                   :resque
+    if @options[:queue_system] == :resque
+      default_processing_queue_to               "CobwebProcessJob"
+      default_crawl_finished_queue_to           "CobwebFinishedJob"
+    else
+      default_processing_queue_to               "CrawlProcessWorker"
+      default_crawl_finished_queue_to           "CrawlFinishedWorker"      
+    end
     default_quiet_to                          true
     default_debug_to                          false
     default_cache_to                          300
@@ -49,7 +55,7 @@ class Cobweb
     default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
     default_valid_mime_types_to                ["*/*"]
     default_raise_exceptions_to               false
-    default_store_refered_url_to              false
+    default_store_inbound_links_to            false
 
   end
   
@@ -80,7 +86,14 @@ class Cobweb
     
     # add internal_urls into redis
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
-    Resque.enqueue(CrawlJob, request)
+    if @options[:queue_system] == :resque
+      Resque.enqueue(CrawlJob, request)
+    elsif @options[:queue_system] == :sidekiq
+      CrawlWorker.perform_async(request)
+    else
+      raise "Unknown queue system: #{content_request[:queue_system]}"
+    end
+    
     request
   end
   
@@ -124,8 +137,13 @@ class Cobweb
 
     # check if it has already been cached
     if ((@options[:cache_type] == :crawl_based && redis.get(unique_id)) || (@options[:cache_type] == :full && full_redis.get(unique_id))) && @options[:cache]
-      puts "Cache hit for #{url}" unless @options[:quiet]
-      content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
+      if @options[:cache_type] == :crawl_based 
+        puts "Cache hit for #{url}" unless @options[:quiet]
+        content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
+      else
+        puts "Cache hit for #{url}" unless @options[:quiet]
+        content = HashUtil.deep_symbolize_keys(Marshal.load(full_redis.get(unique_id)))
+      end
     else
       # retrieve data
       #unless @http && @http.address == uri.host && @http.port == uri.inferred_port
@@ -204,8 +222,13 @@ class Cobweb
         end
         # add content to cache if required
         if @options[:cache]
-          redis.set(unique_id, Marshal.dump(content))
-          redis.expire unique_id, @options[:cache].to_i
+          if @options[:cache_type] == :crawl_based
+            redis.set(unique_id, Marshal.dump(content))
+            redis.expire unique_id, @options[:cache].to_i
+          else
+            full_redis.set(unique_id, Marshal.dump(content))
+            full_redis.expire unique_id, @options[:cache].to_i
+          end
         end
       rescue RedirectError => e
         raise e if @options[:raise_exceptions]

data/lib/cobweb_crawler.rb

@@ -96,14 +96,12 @@ class CobwebCrawler
             @redis.sadd "crawled", url.to_s
             @redis.incr "crawl-counter" 
           
-            internal_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https])
+            document_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https]).uniq
 
             # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
             cobweb_links = CobwebLinks.new(@options)
 
-            internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s) && !cobweb_links.matches_external?(link))}
-
-            all_internal_links = internal_links
+            internal_links = document_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s) && !cobweb_links.matches_external?(link))}
             
             # reject the link if we've crawled it or queued it
             internal_links.reject!{|link| @redis.sismember("crawled", link)}
@@ -120,12 +118,13 @@ class CobwebCrawler
               @queue_counter += 1
             end
 
-            if @options[:store_refered_url]
-              all_internal_links.each do |link|
-                @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(link)}", url)
+            if @options[:store_inbound_links]
+              document_links.each do |target_link|
+                target_uri = UriHelper.parse(target_link)
+                @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(target_uri.to_s)}", UriHelper.parse(url).to_s)
               end
             end
-          
+            
             @crawl_counter = @redis.scard("crawled").to_i
             @queue_counter = @redis.scard("queued").to_i
           

data/lib/cobweb_process_job.rb

@@ -6,7 +6,7 @@ class CobwebProcessJob
 
   # Resque perform method
   def self.perform(content)
-    content = HashHelper.symbolize_keys(content)
+    content = HashUtil.deep_symbolize_keys(content)
     puts "Dummy Processing for #{content[:url]}"
 
     #ap content.keys

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.11"
+    "1.0.12"
   end
 
 end

data/lib/crawl.rb

@@ -88,11 +88,12 @@ module CobwebModule
 
       @cobweb_links = CobwebLinks.new(@options)
       if within_queue_limits?
-        internal_links = ContentLinkParser.new(@options[:url], content.body, @options).all_links(:valid_schemes => [:http, :https])
+        document_links = ContentLinkParser.new(@options[:url], content.body, @options).all_links(:valid_schemes => [:http, :https])
         #get rid of duplicate links in the same page.
-        internal_links.uniq!
+        document_links.uniq!
+        
         # select the link if its internal
-        internal_links.select! { |link| @cobweb_links.internal?(link) }
+        internal_links = document_links.select{ |link| @cobweb_links.internal?(link) }
 
         # reject the link if we've crawled it or queued it
         internal_links.reject! { |link| @redis.sismember("crawled", link) }
@@ -111,6 +112,13 @@ module CobwebModule
             end
           end
         end
+
+        if @options[:store_inbound_links]
+          document_links.each do |link|
+            uri = URI.parse(link)
+            @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(uri.to_s)}", url)
+          end
+        end
       end
     end
 
@@ -180,7 +188,6 @@ module CobwebModule
 
     def finished
       set_first_to_finish
-      debug_ap "CRAWL FINISHED  #{@options[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}"
       @stats.end_crawl(@options)
     end
 

data/lib/crawl_finished_worker.rb

@@ -0,0 +1,27 @@
+require 'sidekiq'
+require File.expand_path(File.dirname(__FILE__) + '/sidekiq/cobweb_helper')
+
+# If your client is single-threaded, we just need a single connection in our Redis connection pool
+#Sidekiq.configure_client do |config|
+#  config.redis = { :namespace => 'x', :size => 1, :url => 'redis://localhost:6379/14' }
+#end
+
+# Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
+#Sidekiq.configure_server do |config|
+#  config.redis = { :namespace => 'x', :url => 'redis://localhost:6379/14' }
+#end
+
+class CrawlFinishedWorker
+  
+  include Sidekiq::Worker
+
+  sidekiq_options queue: "crawl_finished_worker"
+  
+  
+  def perform(statistics)
+    puts "Dummy Finished Job"
+
+    ap statistics
+    
+  end
+end

data/lib/crawl_helper.rb

@@ -0,0 +1,250 @@
+class CrawlHelper
+
+  require "net/https"  
+  require "uri"
+  require "redis"
+  require 'namespaced_redis'
+  
+  def self.crawl_page(content_request)
+    # change all hash keys to symbols
+    content_request = HashUtil.deep_symbolize_keys(content_request)
+    @content_request = content_request
+    
+    content_request[:redis_options] = {} unless content_request.has_key? :redis_options
+    content_request[:crawl_limit_by_page] = false unless content_request.has_key? :crawl_limit_by_page
+    content_request[:valid_mime_types] = ["*/*"] unless content_request.has_key? :valid_mime_types
+    content_request[:queue_system] = content_request[:queue_system].to_sym
+    
+    @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
+    @stats = Stats.new(content_request)
+    
+    @debug = content_request[:debug]
+    
+    decrement_queue_counter
+    
+    # check we haven't crawled this url before
+    unless @redis.sismember "crawled", content_request[:url]
+      # if there is no limit or we're still under it lets get the url
+      if within_crawl_limits?(content_request[:crawl_limit])
+        content = Cobweb.new(content_request).get(content_request[:url], content_request)
+        if content_request[:url] == @redis.get("original_base_url")
+           @redis.set("crawled_base_url", content[:base_url])
+        end
+        if is_permitted_type(content)
+          begin
+            # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
+            @redis.srem "queued", content_request[:url]
+            @redis.sadd "crawled", content_request[:url]
+            @redis.srem "queued", content[:url]
+            @redis.sadd "crawled", content[:url]
+            # increment the counter if we are not limiting by page only || we are limiting count by page and it is a page
+            if content_request[:crawl_limit_by_page]
+              if content[:mime_type].match("text/html")
+                increment_crawl_started_counter
+              end
+            else
+              increment_crawl_started_counter
+            end
+
+            ## update statistics
+            @stats.update_status("Crawling #{content_request[:url]}...")
+            @stats.update_statistics(content)
+
+            # set the base url if this is the first page
+            set_base_url @redis, content, content_request
+
+            @cobweb_links = CobwebLinks.new(content_request)
+            if within_queue_limits?(content_request[:crawl_limit])
+              internal_links = ContentLinkParser.new(content_request[:url], content[:body], content_request).all_links(:valid_schemes => [:http, :https])
+
+              # select the link if its internal
+              internal_links.select! { |link| @cobweb_links.internal?(link) }
+
+              # reject the link if we've crawled it or queued it
+              internal_links.reject! { |link| @redis.sismember("crawled", link) }
+              internal_links.reject! { |link| @redis.sismember("queued", link) }
+
+              internal_links.each do |link|
+                enqueue_content(content_request, link) if within_queue_limits?(content_request[:crawl_limit])
+              end
+            end
+
+            # enqueue to processing queue
+            send_to_processing_queue(content, content_request)
+
+            #if the enqueue counter has been requested update that
+            if content_request.has_key? :enqueue_counter_key
+              enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
+              current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
+              enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
+            end
+
+          ensure
+            #update the queued and crawled lists if we are within the crawl limits.
+
+            # update the queue and crawl counts -- doing this very late in the piece so that the following transaction all occurs at once.
+            # really we should do this with a lock https://github.com/PatrickTulskie/redis-lock
+            if content_request[:crawl_limit_by_page]
+              if content[:mime_type].match("text/html")
+                increment_crawl_counter
+              end
+            else
+              increment_crawl_counter
+            end
+            puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter} In Progress: #{@crawl_started_counter-@crawl_counter}" if @debug
+          end
+        else
+          puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
+        end
+      else
+        puts "ignoring #{content_request[:url]} as outside of crawl limits." if content_request[:debug]
+      end
+      
+    else
+      @redis.srem "queued", content_request[:url]
+      puts "Already crawled #{content_request[:url]}" if content_request[:debug]
+    end
+    
+    # if there's nothing left queued or the crawled limit has been reached
+    refresh_counters
+    if content_request[:crawl_limit].nil? || content_request[:crawl_limit] == 0
+      if @queue_counter+@crawl_started_counter-@crawl_counter == 0
+        finished(content_request)
+      end
+    elsif (@queue_counter +@crawl_started_counter-@crawl_counter)== 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+      finished(content_request)
+    end
+    
+  end
+
+  # Sets the crawl status to 'Crawl Finished' and enqueues the crawl finished job
+  def self.finished(content_request)
+    # finished
+    if @redis.hget("statistics", "current_status")!= "Crawl Finished"
+      ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
+      @stats.end_crawl(content_request)
+      
+      additional_stats = {:crawl_id => content_request[:crawl_id], :crawled_base_url => @redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
+      additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
+      
+      if content_request[:queue_system] == :resque
+        Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge(additional_stats))
+      elsif content_request[:queue_system] == :sidekiq
+        puts "Queueing Finished on Sidekiq"
+        const_get(content_request[:crawl_finished_queue]).perform_async(@stats.get_statistics.merge(additional_stats))
+      else
+        raise "Unknown queue system: #{content_request[:queue_system]}"
+      end
+    else
+      # nothing to report here, we're skipping the remaining urls as we're outside of the crawl limit
+    end
+  end
+  
+  # Enqueues the content to the processing queue setup in options
+  def self.send_to_processing_queue(content, content_request)
+    content_to_send = content.merge({:internal_urls => content_request[:internal_urls], :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
+    if content_request[:direct_call_process_job]
+      clazz = const_get(content_request[:processing_queue])
+      clazz.perform(content_to_send)
+    elsif content_request[:use_encoding_safe_process_job]
+      content_to_send[:body] = Base64.encode64(content[:body])
+      content_to_send[:processing_queue] = content_request[:processing_queue]
+      Resque.enqueue(EncodingSafeProcessJob, content_to_send)
+    else
+      if content_request[:queue_system] == :resque
+        Resque.enqueue(const_get(content_request[:processing_queue]), content_to_send)
+      elsif content_request[:queue_system] == :sidekiq
+        puts "Queueing on Sidekiq"
+        const_get(content_request[:processing_queue]).perform_async(content_to_send)
+      else
+        raise "Unknown queue system: #{content_request[:queue_system]}"
+      end
+    end
+    puts "#{content_request[:url]} has been sent for processing. use_encoding_safe_process_job: #{content_request[:use_encoding_safe_process_job]}" if content_request[:debug]
+  end
+
+  private
+  
+  # Helper method to determine if this content is to be processed or not
+  def self.is_permitted_type(content)
+    @content_request[:valid_mime_types].each do |mime_type|
+      return true if content[:mime_type].match(Cobweb.escape_pattern_for_regex(mime_type))
+    end
+    false
+  end
+  
+  # Returns true if the crawl count is within limits
+  def self.within_crawl_limits?(crawl_limit)
+    refresh_counters
+    crawl_limit.nil? or @crawl_started_counter < crawl_limit.to_i
+  end
+  
+  # Returns true if the queue count is calculated to be still within limits when complete
+  def self.within_queue_limits?(crawl_limit)
+    refresh_counters
+    (@content_request[:crawl_limit_by_page]&& (crawl_limit.nil? or @crawl_counter < crawl_limit.to_i)) || within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (@queue_counter + @crawl_counter) < crawl_limit.to_i)
+  end
+  
+  # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination
+  def self.set_base_url(redis, content, content_request)
+    if redis.get("base_url").nil?
+      unless content[:redirect_through].nil? || content[:redirect_through].empty? || !content_request[:first_page_redirect_internal]
+        uri = Addressable::URI.parse(content[:redirect_through].last)
+        redis.sadd("internal_urls", [uri.scheme, "://", uri.host, "/*"].join)
+      end
+      redis.set("base_url", content[:url])
+    end
+  end
+  
+  # Enqueues content to the crawl_job queue
+  def self.enqueue_content(content_request, link)
+    new_request = content_request.clone
+    new_request[:url] = link
+    new_request[:parent] = content_request[:url]
+    if content_request[:queue_system] == :resque
+      Resque.enqueue(CrawlJob, new_request)
+    elsif content_request[:queue_system] == :sidekiq
+      puts "Queueing content on Sidekiq"
+      CrawlWorker.perform_async(new_request)
+    else
+      raise "Unknown queue system: #{content_request[:queue_system]}"
+    end
+    @redis.sadd "queued", link
+    increment_queue_counter
+  end
+  
+  # Increments the queue counter and refreshes crawl counters
+  def self.increment_queue_counter
+    @redis.incr "queue-counter"
+    refresh_counters
+  end
+  # Increments the crawl counter and refreshes crawl counters
+  def self.increment_crawl_counter
+    @redis.incr "crawl-counter"
+    refresh_counters
+  end
+  def self.increment_crawl_started_counter
+    @redis.incr "crawl-started-counter"
+    refresh_counters
+  end
+  # Decrements the queue counter and refreshes crawl counters
+  def self.decrement_queue_counter
+    @redis.decr "queue-counter"
+    refresh_counters
+  end
+  # Refreshes the crawl counters
+  def self.refresh_counters
+    @crawl_counter = @redis.get("crawl-counter").to_i
+    @crawl_started_counter = @redis.get("crawl-started-counter").to_i
+    @queue_counter = @redis.get("queue-counter").to_i
+  end
+  
+  def self.print_counters
+    puts counters
+  end
+
+  def self.counters
+    "@crawl_counter: #{@crawl_counter} @crawl_started_counter: #{@crawl_started_counter} @queue_counter: #{@queue_counter}"
+  end
+end