cobweb 0.0.29 → 0.0.30

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.29
+h1. Cobweb v0.0.30
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.29"
+    "0.0.30"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -103,6 +103,8 @@ class Cobweb
       uri = Addressable::URI.parse(url.strip)
       
       # retrieve data
+      ap uri.host
+      ap uri.inferred_port
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
         @http = Net::HTTP.new(uri.host, uri.inferred_port)
@@ -112,6 +114,7 @@ class Cobweb
         @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
       
+      ap uri.request_uri
       request_time = Time.now.to_f
       @http.read_timeout = @options[:timeout].to_i
       @http.open_timeout = @options[:timeout].to_i
@@ -125,7 +128,9 @@ class Cobweb
           puts "redirected... " unless @options[:quiet]
           
           # get location to redirect to
-          url = Addressable::URI.parse(response['location']).to_s
+          ap "redirecting to #{response['location']}"
+          url = Addressable::URI.join(uri, response['location']).to_s
+          ap url
           
           # decrement redirect limit
           redirect_limit = redirect_limit - 1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.29
+  version: 0.0.30
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-14 00:00:00.000000000 Z
+date: 2012-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70189333520340 !ruby/object:Gem::Requirement
+  requirement: &70105794152620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333520340
+  version_requirements: *70105794152620
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70189333519900 !ruby/object:Gem::Requirement
+  requirement: &70105794152180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333519900
+  version_requirements: *70105794152180
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70189333519480 !ruby/object:Gem::Requirement
+  requirement: &70105794151760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333519480
+  version_requirements: *70105794151760
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70189333519060 !ruby/object:Gem::Requirement
+  requirement: &70105794151340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333519060
+  version_requirements: *70105794151340
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70189333518640 !ruby/object:Gem::Requirement
+  requirement: &70105794150920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333518640
+  version_requirements: *70105794150920
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70189333518220 !ruby/object:Gem::Requirement
+  requirement: &70105794150500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333518220
+  version_requirements: *70105794150500
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70189333517800 !ruby/object:Gem::Requirement
+  requirement: &70105794166440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333517800
+  version_requirements: *70105794166440
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70189333517380 !ruby/object:Gem::Requirement
+  requirement: &70105794166020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333517380
+  version_requirements: *70105794166020
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70189333533320 !ruby/object:Gem::Requirement
+  requirement: &70105794165600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70189333533320
+  version_requirements: *70105794165600
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com