cobweb 1.0.26 → 1.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b39481d4cdb68d7f602e63a3919e51c126648a2d
-  data.tar.gz: bc29d59ae32beadf047c6e6c73503770d4ca842e
+  metadata.gz: 5cee480edd847679803dcc5e7b47f03523bd5775
+  data.tar.gz: 37ebf10e098d2b46274a80a969962eadea9bd307
 SHA512:
-  metadata.gz: fe824099c8329662036c3df2ec67688ba46f8a3337d63b4942cf93e5dd6de452f3215ddc5a80968dad552e29ff4ba445456112c39cec7e0c1c3166d4aadb91eb
-  data.tar.gz: 43fbe93aa454e77cfdb8f95d4c580dc7a8f6c526025839db75f71ccce02c4f38a2a03e91a2c8417c39311367435c2b6a03168eb48b28eb893014b8e67663ef01
+  metadata.gz: ea3ae531762c16268aac13babd9dbdb482bed8930819de4cdb013f268378eb729483512f181f85ad421a40b19ae2086011d46e6f459286cb314889450be7329c
+  data.tar.gz: ca1bf5d58f6a242af1030fd2d5f83c28444f8eae47e12cb1c6938032687559824924ef313b52e276186c2c480137035258db5154fcf0205445a7b4c811644a14

data/README.textile

@@ -1,4 +1,4 @@
-h1. Cobweb v1.0.26
+h1. Cobweb v1.0.27
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb

data/lib/cobweb.rb

@@ -250,7 +250,10 @@ class Cobweb
           end
         end
       rescue RedirectError => e
-        raise e if @options[:raise_exceptions]
+        if @options[:raise_exceptions]
+          puts "Re-Raising error #{e.message} on #{uri.to_s}"
+          raise e 
+        end
         puts "ERROR RedirectError: #{e.message}"
         
         ## generate a blank content
@@ -453,9 +456,9 @@ class Cobweb
     pattern = pattern.gsub(".", "\\.")
     pattern = pattern.gsub("?", "\\?")
     pattern = pattern.gsub("+", "\\\\+")
-    pattern = pattern.gsub("*", ".*?")
-    if !options.has_key?(:treat_https_as_http) || options[:treat_https_as_http]
-      pattern = pattern.gsub("https", "https?")
+    pattern = pattern.gsub("*", ".*?")  
+    if options[:treat_https_as_http] || !options.has_key?(:treat_https_as_http)
+      pattern = pattern.gsub("http:", "https?:")
     end
     pattern
   end

data/lib/cobweb_crawl_helper.rb

@@ -1,40 +1,54 @@
 # The crawl class gives easy access to information about the crawl, and gives the ability to stop a crawl
 class CobwebCrawlHelper
-  
+
   attr_accessor :id
-  
+
   BATCH_SIZE = 200
   FINISHED = "Finished"
   STARTING = "Starting"
   CANCELLED = "Cancelled"
-  
+
   def initialize(data)
     @data = data
-    
+
     # TAKING A LONG TIME TO RUN ON PRODUCTION BOX
     @stats = Stats.new(data)
   end
-  
+
   def destroy
     options = @data
     options[:queue_name] = "cobweb_crawl_job" unless options.has_key?(:queue_name)
     if RESQUE_INSTALLED
-      options[:finished_resque_queue] = CobwebFinishedJob unless options.has_key?(:finished_resque_queue)
+      options[:processing_queue] = "CobwebJob" unless options.has_key?(:processing_queue)
+      options[:crawl_finished_queue] = "CobwebFinishedJob" unless options.has_key?(:crawl_finished_queue)
+    end
+    if SIDEKIQ_INSTALLED
+      options[:processing_queue] = "CrawlWorker" unless options.has_key?(:processing_queue)
+      options[:crawl_finished_queue] = "CrawlFinishedWorker" unless options.has_key?(:crawl_finished_queue)
     end
-    
+
     # set status as cancelled now so that we don't enqueue any further pages
     self.statistics.end_crawl(@data, true)
-    
 
-    if options[:finished_resque_queue] && options[:queue_system] == :resque && RESQUE_INSTALLED
-      
+
+    if options[:crawl_finished_queue] && options[:queue_system] == :resque && RESQUE_INSTALLED
+
       additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
       additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
       additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
-      
-      Resque.enqueue(options[:finished_resque_queue], @stats.get_statistics.merge(additional_stats))
+
+      Resque.enqueue(options[:crawl_finished_queue], @stats.get_statistics.merge(additional_stats))
     end
-    
+
+    if options[:crawl_finished_queue] && options[:queue_system] == :sidekiq && SIDEKIQ_INSTALLED
+
+      additional_stats = {:crawl_id => id, :crawled_base_url => @stats.redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = @data[:redis_options] unless @data[:redis_options] == {}
+      additional_stats[:source_id] = options[:source_id] unless options[:source_id].nil?
+
+      Kernel.const_get(options[:crawl_finished_queue]).perform_async(@stats.get_statistics.merge(additional_stats))
+    end
+
     counter = 0
     while(counter < 200) do
       break if self.statistics.get_status == CANCELLED
@@ -55,19 +69,27 @@ class CobwebCrawlHelper
         end
       end
     end
-    
+    if options[:queue_system] == :sidekiq && SIDEKIQ_INSTALLED
+      queue_name = Kernel.const_get(options[:processing_queue]).sidekiq_options_hash["queue"]
+      queue = Sidekiq::Queue.new(queue_name)
+      queue.each do |job|
+        job.args # => [1, 2, 3]
+        job.delete if job.args[0]["crawl_id"] == id
+      end
+    end
+
   end
-  
+
   def statistics
     @stats
   end
-  
+
   def status
     statistics.get_status
   end
-  
+
   def id
     @data[:crawl_id]
   end
-  
+
 end

data/lib/cobweb_crawler.rb

@@ -27,7 +27,8 @@ class CobwebCrawler
     @options[:seed_urls].map{|link| @redis.sadd "queued", link }
 
     @options[:crawl_linked_external] = false unless @options.has_key? :crawl_linked_external
-    
+
+    @options[:treat_https_as_http] = true unless @options.has_key? :treat_https_as_http     
     @debug = @options[:debug]
     
     @stats = Stats.new(@options.merge(:crawl_id => @crawl_id))
@@ -100,16 +101,19 @@ class CobwebCrawler
           
             document_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https]).uniq
 
+
             # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
             cobweb_links = CobwebLinks.new(@options)
 
             internal_links = document_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s) && !cobweb_links.matches_external?(link))}
 
             # if the site has the same content for http and https then normalize to http 
+            
             if @options[:treat_https_as_http]
               internal_links.map!{|link| link.gsub(/^https/, "http")}
             end
             
+
             # reject the link if we've crawled it or queued it
             internal_links.reject!{|link| @redis.sismember("crawled", link)}
             internal_links.reject!{|link| @redis.sismember("queued", link)}

data/lib/cobweb_finished_job.rb

@@ -1,6 +1,6 @@
 # Dummy resque job that executes at the end of the crawl if none are specified
 class CobwebFinishedJob
-  require "ap"
+  #require "ap"
   
   @queue = :cobweb_finished_job
 
@@ -8,7 +8,7 @@ class CobwebFinishedJob
   def self.perform(statistics)
     puts "Dummy Finished Job"
 
-    ap statistics
+    #ap statistics
     
   end
 end

data/lib/cobweb_links.rb

@@ -12,9 +12,9 @@ class CobwebLinks
     @options[:external_urls] = [] unless @options.has_key? :external_urls
     @options[:debug] = false unless @options.has_key? :debug
     
-    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, options)}")}
-    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, options)}")}
-    
+    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, @options)}")}
+    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, @options)}")}
+
   end
   
   def allowed?(link)

data/lib/cobweb_process_job.rb

@@ -1,6 +1,6 @@
 # Dummy resque process job that is ran if none are specified
 class CobwebProcessJob
-  require "ap"
+  #require "ap"
   
   @queue = :cobweb_process_job
 

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
 
   # Returns a string of the current version
   def self.version
-    "1.0.26"
+    "1.0.27"
   end
 
 end

data/lib/crawl_finished_worker.rb

@@ -20,7 +20,7 @@ class CrawlFinishedWorker
   def perform(statistics)
     puts "Dummy Finished Job"
 
-    ap statistics
+    #ap statistics
     
   end
 end

data/lib/crawl_helper.rb

@@ -126,7 +126,7 @@ class CrawlHelper
   def self.finished(content_request)
     # finished
     if @redis.hget("statistics", "current_status")!= "Crawl Finished"
-      ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
+      #ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
       @stats.end_crawl(content_request)
 
       additional_stats = {:crawl_id => content_request[:crawl_id], :crawled_base_url => @redis.get("crawled_base_url")}

data/lib/sidekiq/cobweb_helper.rb

@@ -4,14 +4,14 @@ if Gem::Specification.find_all_by_name("sidekiq", ">=1.0.0").count >= 1
   require 'sidekiq'
 else
   SIDEKIQ_INSTALLED = false
-  puts "sidekiq gem not installed, skipping crawl_worker specs"
+  puts "sidekiq gem not installed, skipping crawl_worker specs" if defined?(ENVIRONMENT) && ENVIRONMENT=="test"
 end
 if Gem::Specification.find_all_by_name("resque", ">=1.0.0").count >= 1
   RESQUE_INSTALLED = true
   require 'resque'
 else
   RESQUE_INSTALLED = false
-  puts "resque gem not installed, skipping crawl_job specs"
+  puts "resque gem not installed, skipping crawl_job specs" if defined?(ENVIRONMENT) && ENVIRONMENT=="test"
 end
 
 module Sidekiq

data/spec/cobweb/cobweb_crawl_helper_spec.rb

@@ -3,19 +3,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 describe CobwebCrawlHelper do
   include HttpStubs
   before(:each) do
-    pending("not enabled for non resque installs") unless RESQUE_INSTALLED
+    pending("requires resque or sidekiq") unless RESQUE_INSTALLED || SIDEKIQ_INSTALLED
 
     setup_stubs
   end
   # this spec tests the crawl object
-  
+
   describe "initialize" do
     describe "without data" do
       it "should raise an exception" do
         lambda {CobwebCrawlHelper.new}.should raise_exception
       end
     end
-    
+
     describe "with data" do
       before(:each) do
         data = {:crawl_id => "asdf"}
@@ -30,41 +30,76 @@ describe CobwebCrawlHelper do
       it "should return a status" do
         @crawl.should respond_to "status"
       end
-      
-      describe "the destroy method" do
+
+      describe "the destroy method " do
         before(:each) do
-          if Resque.size("cobweb_crawl_job") > 0
-            raise "cobweb_crawl_job is not empty, do not run specs until it is!"
+          if SIDEKIQ_INSTALLED
+            if Sidekiq::Queue.new("crawl_worker").size > 0
+              raise "cobweb_crawl_job is not empty, do not run specs until it is!"
+            end
+          elsif RESQUE_INSTALLED
+            if Resque.size("cobweb_crawl_job") > 0
+              raise "cobweb_crawl_job is not empty, do not run specs until it is!"
+            end
           end
+
           105.times do |item_count|
             2.times do |crawl_count|
-              item_data = {:crawl_id => "crawl_#{crawl_count}_id", :url => "http://crawl#{crawl_count}.com/page#{item_count}.html"}
-              Resque.enqueue(CrawlJob, item_data)
+              if SIDEKIQ_INSTALLED
+                item_data = {:crawl_id => "crawl_#{crawl_count}_id", :url => "http://crawl#{crawl_count}.com/page#{item_count}.html"}
+                CrawlWorker.perform_async(item_data)
+              elsif RESQUE_INSTALLED
+                item_data = {:crawl_id => "crawl_#{crawl_count}_id", :url => "http://crawl#{crawl_count}.com/page#{item_count}.html"}
+                Resque.enqueue(CrawlJob, item_data)
+              end
             end
           end
         end
         after(:each) do
+          Sidekiq::Queue.new("crawl_worker").clear if SIDEKIQ_INSTALLED
           Resque.remove_queue("cobweb_crawl_job") if RESQUE_INSTALLED
         end
         it "should have a queue length of 210" do
-          Resque.size("cobweb_crawl_job").should == 210
+          Sidekiq::Queue.new("crawl_worker").size.should == 210 if SIDEKIQ_INSTALLED
+          Resque.size("cobweb_crawl_job").should == 210 if RESQUE_INSTALLED
         end
         describe "after called" do
           before(:each) do
-            @crawl = CobwebCrawlHelper.new({:crawl_id => "crawl_0_id", :queue_system => :resque})
+            if SIDEKIQ_INSTALLED
+              @crawl = CobwebCrawlHelper.new({:crawl_id => "crawl_0_id", :queue_system => :sidekiq}) if SIDEKIQ_INSTALLED
+            elsif RESQUE_INSTALLED
+              @crawl = CobwebCrawlHelper.new({:crawl_id => "crawl_0_id", :queue_system => :resque}) if RESQUE_INSTALLED
+            end
             @crawl.destroy
           end
           it "should delete only the crawl specified" do
-            Resque.size("cobweb_crawl_job").should == 105
+            if SIDEKIQ_INSTALLED
+              Sidekiq::Queue.new("crawl_worker").size.should == 105
+            elsif RESQUE_INSTALLED
+              Resque.size("cobweb_crawl_job").should == 105
+            end
+
           end
           it "should not contain any crawl_0_id" do
-            Resque.peek("cobweb_crawl_job", 0, 200).map{|i| i["args"][0]}.each do |item|
-              item["crawl_id"].should_not == "crawl_0_id"
+            if SIDEKIQ_INSTALLED
+              Sidekiq::Queue.new("crawl_job").each do |item|
+                item.args[0]["crawl_id"].should_not == "crawl_0_id"
+              end
+            elsif RESQUE_INSTALLED
+              Resque.peek("cobweb_crawl_job", 0, 200).map{|i| i["args"][0]}.each do |item|
+                item["crawl_id"].should_not == "crawl_0_id"
+              end
             end
           end
           it "should only contain crawl_1_id" do
-            Resque.peek("cobweb_crawl_job", 0, 200).map{|i| i["args"][0]}.each do |item|
-              item["crawl_id"].should == "crawl_1_id"
+            if SIDEKIQ_INSTALLED
+              Sidekiq::Queue.new("crawl_job").each do |item|
+                item.args[0]["crawl_id"].should == "crawl_1_id"
+              end
+            elsif RESQUE_INSTALLED
+              Resque.peek("cobweb_crawl_job", 0, 200).map{|i| i["args"][0]}.each do |item|
+                item["crawl_id"].should == "crawl_1_id"
+              end
             end
           end
           it "should set status to 'Cancelled'" do
@@ -74,6 +109,6 @@ describe CobwebCrawlHelper do
       end
     end
   end
-  
-  
+
+
 end

data/spec/cobweb/cobweb_crawler_spec.rb

@@ -53,6 +53,39 @@ describe CobwebCrawler do
       
     end
 
+    context "internal_links" do
+      it "should match internal links without being explicitly set" do
+        crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1})
+        crawler.crawl(@base_url)
+        queued_links = @redis_mock_object.smembers("queued")
+        queued_links.should_not include("http://themeforest.net/item/cleandream/490140")
+        queued_links.should include("http://localhost:3532/secure")
+      end
+      context "with https" do
+        it "should match https by default" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should include("http://localhost:3532/secure")
+        end
+        it "should not define https as different if treat_https_as_http is true" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1, :treat_https_as_http => true})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should include("http://localhost:3532/secure")
+        end
+        it "should define https as different if treat_https_as_http is false" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1, :treat_https_as_http => false})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should_not include("http://localhost:3532/secure")
+        end
+      end
+    end
+
     context "storing inbound links" do
 
       before(:each) do

data/spec/cobweb/cobweb_spec.rb

@@ -61,19 +61,19 @@ describe Cobweb do
 
     context "with https ignored" do
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com")
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com")
         result.should eql "https?://asdf\\.com"
       end
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com", :treat_https_as_http => true)
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com", :treat_https_as_http => true)
         result.should eql "https?://asdf\\.com"
       end
     end
 
     context "without https ignored" do
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com", :treat_https_as_http => false)
-        result.should eql "https://asdf\\.com"
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com", :treat_https_as_http => false)
+        result.should eql "http://asdf\\.com"
       end
     end
 

data/spec/samples/sample_site/index.html

@@ -711,6 +711,8 @@
 					<a href="gfx/photos/07xl.jpg" class="zoom"><img src="gfx/photos/07.jpg" class="shadow" alt="Photo" /></a>
 					<a href="gfx/photos/08xl.jpg" class="zoom"><img src="gfx/photos/08.jpg" class="shadow" alt="Photo" /></a>
 					<a href="gfx/photos/09xl.jpg" class="zoom"><img src="gfx/photos/09.jpg" class="shadow" alt="Photo" /></a>
+
+					<a href="https://localhost:3532/secure">HTTPS Link</a>
 					
 					<a href="#"><img src="gfx/photos/11.jpg" class="shadow" alt="Photo" /></a>
 					<a href="#"><img src="gfx/photos/12.jpg" class="shadow" alt="Photo" /></a>

data/spec/spec_helper.rb

@@ -4,7 +4,6 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec/samples/sample_serve
 require File.expand_path(File.dirname(__FILE__) + '/../spec/http_stubs')
 require 'mock_redis'
 
-
 require 'coveralls'
 Coveralls.wear!
 
@@ -13,11 +12,11 @@ ENVIRONMENT = "test"
 APP_ROOT = File.expand_path(File.dirname(__FILE__) + '/../')
 
 RSpec.configure do |config|
-  
+
   if ENV["TRAVIS_RUBY_VERSION"] || ENV['CI']
     config.filter_run_excluding :local_only => true
   end
-  
+
   THIN_INSTALLED = false
   if Gem::Specification.find_all_by_name("thin", ">=1.0.0").count >= 1
     require 'thin'
@@ -30,17 +29,18 @@ RSpec.configure do |config|
   # WAIT FOR START TO COMPLETE
   sleep 1
 
-  
+
   config.before(:all) {
     # START THIN SERVER TO HOST THE SAMPLE SITE FOR CRAWLING
   }
-  
+
   config.before(:each) {
         
-    #redis_mock = double("redis")
-    #redis_mock.stub(:new).and_return(@redis_mock_object)
+    @redis_mock_object = MockRedis.new
+    Redis.stub(:new).and_return(@redis_mock_object)
+    Redis::Namespace.stub(:new).and_return(@redis_mock_object)
     
-    #redis_mock.flushdb
+    @redis_mock_object.flushdb
     
   }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.26
+  version: 1.0.27
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-15 00:00:00.000000000 Z
+date: 2015-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis