cobweb 1.0.5 → 1.0.6

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.5
+h1. Cobweb v1.0.6
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 

data/lib/cobweb.rb

@@ -48,6 +48,7 @@ class Cobweb
     default_user_agent_to                     "cobweb/#{Cobweb.version} (ruby/#{RUBY_VERSION} nokogiri/#{Nokogiri::VERSION})"
     default_valid_mime_types_to                ["*/*"]
     default_raise_exceptions_to               false
+    default_store_refered_url_to              false
     
   end
   
@@ -139,19 +140,19 @@ class Cobweb
       @http.read_timeout = @options[:timeout].to_i
       @http.open_timeout = @options[:timeout].to_i
       begin
-        puts "Retrieving #{url }... " unless @options[:quiet]
+        puts "Retrieving #{uri}... " unless @options[:quiet]
         request_options={}
-        request_options['Cookie']= options[:cookies] if options.has_key?(:cookies)
+        request_options['Cookie']= options[:cookies] if options[:cookies]
         request_options['User-Agent']= options[:user_agent] if options.has_key?(:user_agent)
 
         request = Net::HTTP::Get.new uri.request_uri, request_options
         response = @http.request request
 
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
-          puts "redirected... " unless @options[:quiet]
 
           # get location to redirect to
           uri = UriHelper.join_no_fragment(uri, response['location'])
+          puts "Following Redirect to #{uri}... " unless @options[:quiet]
 
           # decrement redirect limit
           redirect_limit = redirect_limit - 1
@@ -398,7 +399,7 @@ class Cobweb
     end
     
   end
-  
+
   # escapes characters with meaning in regular expressions and adds wildcard expression
   def self.escape_pattern_for_regex(pattern)
     pattern = pattern.gsub(".", "\\.")

data/lib/cobweb_crawler.rb

@@ -54,6 +54,7 @@ class CobwebCrawler
       while queue_counter>0 && (@options[:crawl_limit].to_i == 0 || @options[:crawl_limit].to_i > crawl_counter)      
         thread = Thread.new do
         
+
           url = @redis.spop "queued"
           queue_counter = 0 if url.nil?
 
@@ -76,6 +77,8 @@ class CobwebCrawler
                 cobweb_links = CobwebLinks.new(@options)
 
                 internal_links = internal_links.select{|link| cobweb_links.internal?(link) || (@options[:crawl_linked_external] && cobweb_links.internal?(url.to_s))}
+
+                all_internal_links = internal_links
                 
                 # reject the link if we've crawled it or queued it
                 internal_links.reject!{|link| @redis.sismember("crawled", link)}
@@ -91,6 +94,12 @@ class CobwebCrawler
                   @redis.hset "navigation", url, children
                   queue_counter += 1
                 end
+
+                if @options[:store_refered_url]
+                  all_internal_links.each do |link|
+                    @redis.sadd("inbound_links_#{Digest::MD5.hexdigest(link)}", url)
+                  end
+                end
               
                 crawl_counter = @redis.scard("crawled").to_i
                 queue_counter = @redis.scard("queued").to_i
@@ -117,7 +126,7 @@ class CobwebCrawler
     ensure
       @stats.end_crawl(@options)
     end
-    @stats.get_statistics
+    @stats
   end
   
 end

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.5"
+    "1.0.6"
   end
 
 end

data/lib/stats.rb

@@ -36,7 +36,11 @@ class Stats
   def get_crawled
     @redis.smembers "crawled"
   end
-  
+
+  def inbound_links_for(url, redis=@redis)
+    @redis.smembers("inbound_links_#{Digest::MD5.hexdigest(url)}")
+  end
+
   # Returns statistics hash.  update_statistics takes the content hash, extracts statistics from it and updates redis with the data.  
   def update_statistics(content, crawl_counter=@redis.scard("crawled").to_i, queue_counter=@redis.scard("queued").to_i)
     

data/spec/cobweb/cobweb_crawler_spec.rb

@@ -32,7 +32,7 @@ describe CobwebCrawler do
       statistics = crawler.crawl(@base_url)
       
       statistics.should_not be_nil
-      statistics.should be_an_instance_of Hash
+      statistics.get_statistics.should be_an_instance_of Hash
       
     end
     
@@ -48,7 +48,7 @@ describe CobwebCrawler do
       end
       
       statistics.should_not be_nil
-      statistics.should be_an_instance_of Hash
+      statistics.get_statistics.should be_an_instance_of Hash
       
     end
   end  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-08 00:00:00.000000000 Z
+date: 2013-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050084220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,15 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050084220
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050081420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -37,15 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050081420
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050075780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -53,15 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050075780
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050066140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -69,15 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050066140
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050054340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -85,15 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050054340
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050049640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -101,15 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050049640
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050048160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -117,15 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050048160
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050047400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -133,15 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050047400
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050046440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -149,15 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050046440
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050045060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -165,15 +120,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050045060
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70331050044240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -181,12 +131,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70331050044240
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface
@@ -569,7 +514,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly