cobweb 1.0.24 → 1.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b19a42aa5998a48743870fb7c0ed4e02ab0109f3
-  data.tar.gz: ad64a73409057595e0b60ab9cdf0d1b8e70339e6
+  metadata.gz: 433da726316611ac2835723ff04e645fb00a3dc4
+  data.tar.gz: e85d23955c5ddbb02cf69baef0c5e240ff7d8204
 SHA512:
-  metadata.gz: c91107bc0bb4cf6257b4707cbaaee723ad898750150404863715edc0ce9619bf7c0156ffd292758d5dccadf641a0b8d24347569d443d11c8c42d5c9ff446ffa9
-  data.tar.gz: 61fa0d909f9c2763d04fcd189e0b47ed55fe7290726ff7d1b72c7c902e409612e7d9e64a800b41f046c1682c1f4ced18442ab98d2752ba0057fd24988a738ff8
+  metadata.gz: 23b0e6707b07bcad8621f8c547f48c3c8d0abf946e6454ffa3fee599fdfac5ab01942579c3f4a16ce9e743034225e68695f77df752032acc0c9fbfdfeb7e43ce
+  data.tar.gz: 702b5ed7c93e56f3994c7bee735ef1a44ceabd0cf349e1976b3692a094465b949ca618c106f1896e33551caeb64fe5383b57f53b8a8b4f18144b41255c092ad7

data/README.textile

@@ -1,4 +1,4 @@
-h1. Cobweb v1.0.24
+h1. Cobweb v1.0.25
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb

data/lib/cobweb.rb

@@ -146,7 +146,7 @@ class Cobweb
     content = {:base_url => url}
 
     # check if it has already been cached
-    if ((@options[:cache_type] == :crawl_based && redis.get(unique_id)) || (@options[:cache_type] == :full && full_redis.get(unique_id))) && @options[:cache]
+    if @options[:cache] && ((@options[:cache_type] == :crawl_based && redis.get(unique_id)) || (@options[:cache_type] == :full && full_redis.get(unique_id)))
       if @options[:cache_type] == :crawl_based 
         puts "Cache hit in crawl for #{url}" unless @options[:quiet]
         content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
@@ -250,7 +250,10 @@ class Cobweb
           end
         end
       rescue RedirectError => e
-        raise e if @options[:raise_exceptions]
+        if @options[:raise_exceptions]
+          puts "Re-Raising error #{e.message} on #{uri.to_s}"
+          raise e 
+        end
         puts "ERROR RedirectError: #{e.message}"
         
         ## generate a blank content
@@ -327,7 +330,7 @@ class Cobweb
     content = {:base_url => url}
     
     # check if it has already been cached
-    if redis.get("head-#{unique_id}") and @options[:cache]
+    if @options[:cache] && redis.get("head-#{unique_id}")
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get("head-#{unique_id}")))
     else
@@ -453,9 +456,9 @@ class Cobweb
     pattern = pattern.gsub(".", "\\.")
     pattern = pattern.gsub("?", "\\?")
     pattern = pattern.gsub("+", "\\\\+")
-    pattern = pattern.gsub("*", ".*?")
-    if !options.has_key?(:treat_https_as_http) || options[:treat_https_as_http]
-      pattern = pattern.gsub("https", "https?")
+    pattern = pattern.gsub("*", ".*?")  
+    if options[:treat_https_as_http] || !options.has_key?(:treat_https_as_http)
+      pattern = pattern.gsub("http:", "https?:")
     end
     pattern
   end

data/lib/cobweb_crawler.rb

@@ -27,7 +27,8 @@ class CobwebCrawler
     @options[:seed_urls].map{|link| @redis.sadd "queued", link }
 
     @options[:crawl_linked_external] = false unless @options.has_key? :crawl_linked_external
-    
+
+    @options[:treat_https_as_http] = true unless @options.has_key? :treat_https_as_http     
     @debug = @options[:debug]
     
     @stats = Stats.new(@options.merge(:crawl_id => @crawl_id))
@@ -100,16 +101,19 @@ class CobwebCrawler
           
             document_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https]).uniq
 
+
             # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
             cobweb_links = CobwebLinks.new(@options)
 
             internal_links = document_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s) && !cobweb_links.matches_external?(link))}
 
             # if the site has the same content for http and https then normalize to http 
+            
             if @options[:treat_https_as_http]
               internal_links.map!{|link| link.gsub(/^https/, "http")}
             end
             
+
             # reject the link if we've crawled it or queued it
             internal_links.reject!{|link| @redis.sismember("crawled", link)}
             internal_links.reject!{|link| @redis.sismember("queued", link)}

data/lib/cobweb_links.rb

@@ -12,9 +12,9 @@ class CobwebLinks
     @options[:external_urls] = [] unless @options.has_key? :external_urls
     @options[:debug] = false unless @options.has_key? :debug
     
-    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, options)}")}
-    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, options)}")}
-    
+    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, @options)}")}
+    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{Cobweb.escape_pattern_for_regex(pattern, @options)}")}
+
   end
   
   def allowed?(link)

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
 
   # Returns a string of the current version
   def self.version
-    "1.0.24"
+    "1.0.25"
   end
 
 end

data/lib/sidekiq/cobweb_helper.rb

@@ -4,14 +4,14 @@ if Gem::Specification.find_all_by_name("sidekiq", ">=1.0.0").count >= 1
   require 'sidekiq'
 else
   SIDEKIQ_INSTALLED = false
-  puts "sidekiq gem not installed, skipping crawl_worker specs"
+  puts "sidekiq gem not installed, skipping crawl_worker specs" if defined?(ENVIRONMENT) && ENVIRONMENT=="test"
 end
 if Gem::Specification.find_all_by_name("resque", ">=1.0.0").count >= 1
   RESQUE_INSTALLED = true
   require 'resque'
 else
   RESQUE_INSTALLED = false
-  puts "resque gem not installed, skipping crawl_job specs"
+  puts "resque gem not installed, skipping crawl_job specs" if defined?(ENVIRONMENT) && ENVIRONMENT=="test"
 end
 
 module Sidekiq

data/spec/cobweb/cobweb_crawler_spec.rb

@@ -53,6 +53,39 @@ describe CobwebCrawler do
       
     end
 
+    context "internal_links" do
+      it "should match internal links without being explicitly set" do
+        crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1})
+        crawler.crawl(@base_url)
+        queued_links = @redis_mock_object.smembers("queued")
+        queued_links.should_not include("http://themeforest.net/item/cleandream/490140")
+        queued_links.should include("http://localhost:3532/secure")
+      end
+      context "with https" do
+        it "should match https by default" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should include("http://localhost:3532/secure")
+        end
+        it "should not define https as different if treat_https_as_http is true" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1, :treat_https_as_http => true})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should include("http://localhost:3532/secure")
+        end
+        it "should define https as different if treat_https_as_http is false" do
+          crawler = CobwebCrawler.new({:cache => false, :crawl_limit => 1, :treat_https_as_http => false})
+          crawler.crawl(@base_url)
+          queued_links = @redis_mock_object.smembers("queued")
+          queued_links.should_not include("https://localhost:3532/secure")
+          queued_links.should_not include("http://localhost:3532/secure")
+        end
+      end
+    end
+
     context "storing inbound links" do
 
       before(:each) do

data/spec/cobweb/cobweb_spec.rb

@@ -61,19 +61,19 @@ describe Cobweb do
 
     context "with https ignored" do
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com")
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com")
         result.should eql "https?://asdf\\.com"
       end
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com", :treat_https_as_http => true)
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com", :treat_https_as_http => true)
         result.should eql "https?://asdf\\.com"
       end
     end
 
     context "without https ignored" do
       it "should ignore https" do
-        result = Cobweb.escape_pattern_for_regex("https://asdf.com", :treat_https_as_http => false)
-        result.should eql "https://asdf\\.com"
+        result = Cobweb.escape_pattern_for_regex("http://asdf.com", :treat_https_as_http => false)
+        result.should eql "http://asdf\\.com"
       end
     end
 

data/spec/samples/sample_site/index.html

@@ -711,6 +711,8 @@
 					<a href="gfx/photos/07xl.jpg" class="zoom"><img src="gfx/photos/07.jpg" class="shadow" alt="Photo" /></a>
 					<a href="gfx/photos/08xl.jpg" class="zoom"><img src="gfx/photos/08.jpg" class="shadow" alt="Photo" /></a>
 					<a href="gfx/photos/09xl.jpg" class="zoom"><img src="gfx/photos/09.jpg" class="shadow" alt="Photo" /></a>
+
+					<a href="https://localhost:3532/secure">HTTPS Link</a>
 					
 					<a href="#"><img src="gfx/photos/11.jpg" class="shadow" alt="Photo" /></a>
 					<a href="#"><img src="gfx/photos/12.jpg" class="shadow" alt="Photo" /></a>

data/spec/spec_helper.rb

@@ -37,10 +37,11 @@ RSpec.configure do |config|
   
   config.before(:each) {
         
-    #redis_mock = double("redis")
-    #redis_mock.stub(:new).and_return(@redis_mock_object)
+    @redis_mock_object = MockRedis.new
+    Redis.stub(:new).and_return(@redis_mock_object)
+    Redis::Namespace.stub(:new).and_return(@redis_mock_object)
     
-    #redis_mock.flushdb
+    @redis_mock_object.flushdb
     
   }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.24
+  version: 1.0.25
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-22 00:00:00.000000000 Z
+date: 2015-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis