cobweb 1.0.1 → 1.0.2

data/README.textile

@@ -1,10 +1,10 @@
 
-h1. Cobweb v1.0.1
+h1. Cobweb v1.0.2
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
-
+!https://gemnasium.com/stewartmckee/cobweb.png!
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -38,6 +38,7 @@ class Cobweb
     default_quiet_to                          true
     default_debug_to                          false
     default_cache_to                          300
+    default_cache_type_to                     :crawl_based # other option is :full
     default_timeout_to                        10
     default_redis_options_to                  Hash.new
     default_internal_urls_to                  []
@@ -114,11 +115,12 @@ class Cobweb
     else
       redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
     end
+    full_redis = Redis::Namespace.new("cobweb-#{Cobweb.version}", :redis => Redis.new(@options[:redis_options]))
 
     content = {:base_url => url}
 
     # check if it has already been cached
-    if redis.get(unique_id) and @options[:cache]
+    if ((@options[:cache_type] == :crawl_based && redis.get(unique_id)) || (@options[:cache_type] == :full && full_redis.get(unique_id))) && @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
     else
@@ -138,9 +140,9 @@ class Cobweb
       begin
         print "Retrieving #{url }... " unless @options[:quiet]
         request_options={}
-        if options[:cookies]
-          request_options[ 'Cookie']= options[:cookies]
-        end
+        request_options['Cookie']= options[:cookies] if options.has_key?(:cookies)
+        request_options['User-Agent']= options[:user_agent] if options.has_key?(:user_agent)
+
         request = Net::HTTP::Get.new uri.request_uri, request_options
 
         response = @http.request request

data/lib/cobweb_crawler.rb

@@ -23,6 +23,9 @@ class CobwebCrawler
     @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@crawl_id}", :redis => Redis.new(@options[:redis_options]))
     @options[:internal_urls] = [] if @options[:internal_urls].nil?
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
+
+    @options[:crawl_linked_external] = false unless @options.has_key? :crawl_linked_external
+    
     @debug = @options[:debug]
     
     @stats = Stats.new(@options.merge(:crawl_id => @crawl_id))
@@ -52,12 +55,13 @@ class CobwebCrawler
         thread = Thread.new do
         
           url = @redis.spop "queued"
-        
+          queue_counter = 0 if url.nil?
+
           @options[:url] = url
           unless @redis.sismember("crawled", url.to_s)
             begin
               @stats.update_status("Requesting #{url}...")
-              content = @cobweb.get(url)
+              content = @cobweb.get(url) unless url.nil?
               if content.nil?
                 queue_counter = queue_counter - 1 #@redis.scard("queued").to_i
               else
@@ -70,8 +74,9 @@ class CobwebCrawler
 
                 # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
                 cobweb_links = CobwebLinks.new(@options)
-                internal_links = internal_links.select{|link| cobweb_links.internal?(link)}
-              
+
+                internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s))}
+                
                 # reject the link if we've crawled it or queued it
                 internal_links.reject!{|link| @redis.sismember("crawled", link)}
                 internal_links.reject!{|link| @redis.sismember("queued", link)}
@@ -95,9 +100,13 @@ class CobwebCrawler
                 yield content, @stats.get_statistics if block_given?
               end
             rescue => e
-              puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
-              ap e
-              ap e.backtrace
+              puts "Error loading #{url}: #{e}"
+              #puts "!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!"
+              #ap e
+              #ap e.backtrace
+            ensure
+              crawl_counter = @redis.scard("crawled").to_i
+              queue_counter = @redis.scard("queued").to_i
             end
           else
             puts "Already crawled #{@options[:url]}" if @debug

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.1"
+    "1.0.2"
   end
 
 end

data/lib/robots.rb

@@ -8,12 +8,12 @@ class Robots
     raise ":url is required" unless @options.has_key? :url
     @options[:file] = "robots.txt" unless @options.has_key? :file
     @options[:user_agent] = "cobweb" unless @options.has_key? :user_agent
-    
+
     uri = URI.parse(@options[:url])
     content = Cobweb.new(:cache => nil, :text_mime_types => ["text/html", "application/xhtml+xml", "text/plain"]).get([uri.scheme, "://", uri.host, ":", uri.port, "/", @options[:file]].join)
     if content[:mime_type][0..4] == "text/"
       @raw_data = parse_data(content[:body])
-      
+
       if @options.has_key?(:user_agent) && @raw_data.has_key?(@options[:user_agent].to_s.downcase.to_sym)
         @params = @raw_data[@options[:user_agent].to_s.downcase.to_sym]
       else

data/spec/cobweb/cobweb_spec.rb

@@ -186,7 +186,7 @@ describe Cobweb do
     describe "location setting" do
       it "Get should strip fragments" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/", {})
+        Net::HTTP::Get.should_receive(:new).with("/", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
         @cobweb.get("http://www.google.com/#ignore")
       end
       it "head should strip fragments" do
@@ -196,12 +196,12 @@ describe Cobweb do
       end
       it "get should not strip path" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", {})
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
         @cobweb.get("http://www.google.com/path/to/stuff#ignore")
       end
       it "get should not strip query string" do
         Net::HTTP.should_receive(:new).with("www.google.com", 80)
-        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", {})
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"})
         @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
       end
     end

data/spec/cobweb/robots_spec.rb

@@ -8,7 +8,7 @@ describe Robots do
   
   describe "default user-agent" do
     before(:each) do
-      @options = {:url => "http://localhost/"}
+      @options = {:url => "http://localhost:3532/"}
     end
     
     it "should parse a valid robots.txt" do

data/spec/spec_helper.rb

@@ -63,9 +63,9 @@ RSpec.configure do |config|
     
     Net::HTTP.stub!(:new).and_return(@mock_http_client)
     Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect.html", {}).and_return(@mock_http_redirect_request)
-    Net::HTTP::Get.stub!(:new).with("/robots.txt", {}).and_return(@mock_http_robot_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect2.html", {}).and_return(@mock_http_redirect_request2)
+    Net::HTTP::Get.stub!(:new).with("/redirect.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub!(:new).with("/robots.txt", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_robot_request)
+    Net::HTTP::Get.stub!(:new).with("/redirect2.html", {"User-Agent"=>"cobweb/#{CobwebVersion.version} (ruby/#{RUBY_VERSION} nokogiri/1.5.0)"}).and_return(@mock_http_redirect_request2)
 
     Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-19 00:00:00.000000000 Z
+date: 2012-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70296707613060 !ruby/object:Gem::Requirement
+  requirement: &70096521975680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707613060
+  version_requirements: *70096521975680
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70296707611440 !ruby/object:Gem::Requirement
+  requirement: &70096521974080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707611440
+  version_requirements: *70096521974080
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70296707610120 !ruby/object:Gem::Requirement
+  requirement: &70096521972700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707610120
+  version_requirements: *70096521972700
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70296707609180 !ruby/object:Gem::Requirement
+  requirement: &70096521971740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707609180
+  version_requirements: *70096521971740
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70296707608660 !ruby/object:Gem::Requirement
+  requirement: &70096521971220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707608660
+  version_requirements: *70096521971220
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70296707608040 !ruby/object:Gem::Requirement
+  requirement: &70096521970620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707608040
+  version_requirements: *70096521970620
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70296707607220 !ruby/object:Gem::Requirement
+  requirement: &70096521969800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707607220
+  version_requirements: *70096521969800
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70296707606660 !ruby/object:Gem::Requirement
+  requirement: &70096521969240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707606660
+  version_requirements: *70096521969240
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70296707606220 !ruby/object:Gem::Requirement
+  requirement: &70096521968800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707606220
+  version_requirements: *70096521968800
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70296707605480 !ruby/object:Gem::Requirement
+  requirement: &70096521968060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +120,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70296707605480
+  version_requirements: *70096521968060
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70296707604780 !ruby/object:Gem::Requirement
+  requirement: &70096521967360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,7 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70296707604780
+  version_requirements: *70096521967360
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface