cobweb 0.0.61 → 0.0.62

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.61
+h1. Cobweb v0.0.62
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 

data/lib/cobweb.rb

@@ -66,6 +66,7 @@ class Cobweb
     
     request.merge!(@options)
     @redis = NamespacedRedis.new(request[:redis_options], "cobweb-#{Cobweb.version}-#{request[:crawl_id]}")
+    @redis.set("original_base_url", base_url)
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
     @redis.set("queue-counter", 1)
@@ -75,7 +76,6 @@ class Cobweb
     
     # add internal_urls into redis
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
-    
     Resque.enqueue(CrawlJob, request)
     request
   end
@@ -114,7 +114,7 @@ class Cobweb
     else
       redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
     end
-    
+
     content = {:base_url => url}
   
     # check if it has already been cached
@@ -149,7 +149,7 @@ class Cobweb
           puts "redirected... " unless @options[:quiet]
           
           # get location to redirect to
-          url = UriHelper.join_no_fragment(uri, response['location'])
+          uri = UriHelper.join_no_fragment(uri, response['location'])
           
           # decrement redirect limit
           redirect_limit = redirect_limit - 1
@@ -158,7 +158,7 @@ class Cobweb
           cookies = get_cookies(response)
 
           # get the content from redirect location
-          content = get(url, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
+          content = get(uri, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)
@@ -307,14 +307,14 @@ class Cobweb
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
           puts "redirected... " unless @options[:quiet]
 
-          url = UriHelper.join_no_fragment(uri, response['location'])
+          uri = UriHelper.join_no_fragment(uri, response['location'])
 
           redirect_limit = redirect_limit - 1
 
           raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
           cookies = get_cookies(response)
 
-          content = head(url, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
+          content = head(uri, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.61"
+    "0.0.62"
   end
 
 end

data/lib/crawl_job.rb

@@ -28,9 +28,11 @@ class CrawlJob
     
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
-      
       content = Cobweb.new(content_request).get(content_request[:url], content_request)
-
+      if content_request[:url] == @redis.get("original_base_url")
+        puts content
+         @redis.set("crawled_base_url", content[:base_url])
+      end
       if is_permitted_type(content)
         # if there is no limit or we're still under it lets get the url
         if within_crawl_limits?(content_request[:crawl_limit])
@@ -124,9 +126,9 @@ class CrawlJob
   def self.finished(content_request)
     # finished
     if @redis.hget("statistics", "current_status")!= "Crawl Stopped"
-      ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}" if content_request[:debug]
+      ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
       @stats.end_crawl(content_request)
-      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id]}))
+      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id], :crawled_base_url => @redis.get("crawled_base_url")}))
     else
       ap "CRAWL REFINISHED  #{content_request[:url]}, #{counters}" if content_request[:debug]
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.61
+  version: 0.0.62
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-17 00:00:00.000000000 Z
+date: 2012-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70301996170680 !ruby/object:Gem::Requirement
+  requirement: &70305062213520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996170680
+  version_requirements: *70305062213520
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70301996168960 !ruby/object:Gem::Requirement
+  requirement: &70305062213040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996168960
+  version_requirements: *70305062213040
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70301996168220 !ruby/object:Gem::Requirement
+  requirement: &70305062212160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996168220
+  version_requirements: *70305062212160
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70301996167040 !ruby/object:Gem::Requirement
+  requirement: &70305062211220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996167040
+  version_requirements: *70305062211220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70301996165600 !ruby/object:Gem::Requirement
+  requirement: &70305062210040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996165600
+  version_requirements: *70305062210040
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70301996180600 !ruby/object:Gem::Requirement
+  requirement: &70305062208860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996180600
+  version_requirements: *70305062208860
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70301996179980 !ruby/object:Gem::Requirement
+  requirement: &70305062224320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996179980
+  version_requirements: *70305062224320
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70301996179420 !ruby/object:Gem::Requirement
+  requirement: &70305062223780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996179420
+  version_requirements: *70305062223780
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70301996178900 !ruby/object:Gem::Requirement
+  requirement: &70305062223120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70301996178900
+  version_requirements: *70305062223120
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70301996178200 !ruby/object:Gem::Requirement
+  requirement: &70305062222400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70301996178200
+  version_requirements: *70305062222400
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface