cobweb 1.0.2 → 1.0.3

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.2
+h1. Cobweb v1.0.3
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 
@@ -101,6 +101,7 @@ Creates a new crawler object based on a base_url
     ** :crawl_limit_by_page           - sets the crawl counter to only use html page types when counting objects crawled
     ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to ['*/*']
     ** :direct_call_process_job       - boolean that specifies whether objects should be passed directly to a processing method or should be put onto a queue
+    ** :raise_exceptions              - defaults to handling exceptions with debug output, setting this to true will raise exceptions in your app
     
     
 bc. crawler = Cobweb.new(:follow_redirects => false)

data/lib/cobweb.rb

@@ -47,6 +47,7 @@ class Cobweb
     default_obey_robots_to                    false
     default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
     default_valid_mime_types_to                ["*/*"]
+    default_raise_exceptions_to               false
     
   end
   
@@ -205,6 +206,7 @@ class Cobweb
           redis.expire unique_id, @options[:cache].to_i
         end
       rescue RedirectError => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR RedirectError: #{e.message}"
         
         ## generate a blank content
@@ -220,6 +222,7 @@ class Cobweb
         content[:links] = {}
         
       rescue SocketError => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR SocketError: #{e.message}"
         
         ## generate a blank content
@@ -235,6 +238,7 @@ class Cobweb
         content[:links] = {}
         
       rescue Timeout::Error => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR Timeout::Error: #{e.message}"
         
         ## generate a blank content
@@ -342,6 +346,7 @@ class Cobweb
           end
         end
       rescue RedirectError => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR RedirectError: #{e.message}"
 
         ## generate a blank content
@@ -357,6 +362,7 @@ class Cobweb
         content[:links] = {}
 
       rescue SocketError => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR SocketError: #{e.message}"
         
         ## generate a blank content
@@ -372,6 +378,7 @@ class Cobweb
         content[:links] = {}
         
       rescue Timeout::Error => e
+        raise e if @options[:raise_exceptions]
         puts "ERROR Timeout::Error: #{e.message}"
         
         ## generate a blank content

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.2"
+    "1.0.3"
   end
 
 end

data/spec/cobweb/cobweb_spec.rb

@@ -5,6 +5,8 @@ describe Cobweb do
   before(:each) do
     @base_url = "http://www.baseurl.com/"
     @cobweb = Cobweb.new :quiet => true, :cache => nil
+
+    @default_options = {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"}
   end
   
   it "should generate a cobweb object" do
@@ -186,7 +188,7 @@ describe Cobweb do
     describe "location setting" do
       it "Get should strip fragments" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
+        Net::HTTP::Get.should_receive(:new).with("/", @default_options)
         @cobweb.get("http://www.google.com/#ignore")
       end
       it "head should strip fragments" do
@@ -196,12 +198,12 @@ describe Cobweb do
       end
       it "get should not strip path" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff#ignore")
       end
       it "get should not strip query string" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
       end
     end

data/spec/spec_helper.rb

@@ -63,9 +63,9 @@ RSpec.configure do |config|
     
     Net::HTTP.stub!(:new).and_return(@mock_http_client)
     Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_redirect_request)
-    Net::HTTP::Get.stub!(:new).with("/robots.txt", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_robot_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect2.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_redirect_request2)
+    Net::HTTP::Get.stub!(:new).with("/redirect.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"}).and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub!(:new).with("/robots.txt", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"}).and_return(@mock_http_robot_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect2.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"}).and_return(@mock_http_redirect_request2)
 
     Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-28 00:00:00.000000000 Z
+date: 2013-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70096521975680 !ruby/object:Gem::Requirement
+  requirement: &70115295148600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521975680
+  version_requirements: *70115295148600
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70096521974080 !ruby/object:Gem::Requirement
+  requirement: &70115295147540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521974080
+  version_requirements: *70115295147540
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70096521972700 !ruby/object:Gem::Requirement
+  requirement: &70115295145880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521972700
+  version_requirements: *70115295145880
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70096521971740 !ruby/object:Gem::Requirement
+  requirement: &70115295144920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521971740
+  version_requirements: *70115295144920
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70096521971220 !ruby/object:Gem::Requirement
+  requirement: &70115295144060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521971220
+  version_requirements: *70115295144060
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70096521970620 !ruby/object:Gem::Requirement
+  requirement: &70115295143400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521970620
+  version_requirements: *70115295143400
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70096521969800 !ruby/object:Gem::Requirement
+  requirement: &70115295142120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521969800
+  version_requirements: *70115295142120
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70096521969240 !ruby/object:Gem::Requirement
+  requirement: &70115295138080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521969240
+  version_requirements: *70115295138080
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70096521968800 !ruby/object:Gem::Requirement
+  requirement: &70115295134720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,21 +109,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521968800
+  version_requirements: *70115295134720
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70096521968060 !ruby/object:Gem::Requirement
+  requirement: &70115295131880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521968060
+  version_requirements: *70115295131880
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70096521967360 !ruby/object:Gem::Requirement
+  requirement: &70115295127820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70096521967360
+  version_requirements: *70115295127820
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface