spider 0.4.2 → 0.4.3

data/CHANGES

@@ -1,3 +1,6 @@
+2008-10-09
+* fixed a situation with nested slashes in urls, thanks to Sander van der Vliet and John Buckley
+
 2008-07-06
 * Trap interrupts and shutdown gracefully
 * Support for custom urls-to-crawl objects

data/README

@@ -1,3 +1,4 @@
+
 Spider, a Web spidering library for Ruby. It handles the robots.txt,
 scraping, collecting, and looping so that you can just handle the data.
 
@@ -132,9 +133,14 @@ scraping, collecting, and looping so that you can just handle the data.
 == Author
 
 John Nagro john.nagro@gmail.com
+
 Mike Burns http://mike-burns.com mike@mike-burns.com (original author)
 
-Help from Matt Horan, and Henri Cook.
+Many thanks to:
+Matt Horan
+Henri Cook
+Sander van der Vliet
+John Buckley
 
 With `robot_rules' from James Edward Gray II via
 http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/177589

data/lib/spider/spider_instance.rb

@@ -295,15 +295,17 @@ class SpiderInstance
   def construct_complete_url(base_url, additional_url, parsed_additional_url = nil) #:nodoc:
     parsed_additional_url ||= URI.parse(additional_url)
     case parsed_additional_url.scheme
-    when nil
-      u = base_url.is_a?(URI) ? base_url : URI.parse(base_url)
-      if additional_url[0].chr == '/'
-        "#{u.scheme}://#{u.host}:#{u.port}#{additional_url}"
-      elsif u.path.nil? || u.path == ''
-        "#{u.scheme}://#{u.host}:#{u.port}/#{additional_url}"
-      else
-        "#{u.scheme}://#{u.host}:#{u.port}/#{u.path}/#{additional_url}"
-      end
+      when nil
+        u = base_url.is_a?(URI) ? base_url : URI.parse(base_url)
+        if additional_url[0].chr == '/'
+          "#{u.scheme}://#{u.host}#{additional_url}"
+        elsif u.path.nil? || u.path == ''
+          "#{u.scheme}://#{u.host}/#{additional_url}"
+        elsif u.path[0].chr == '/'
+          "#{u.scheme}://#{u.host}#{u.path}/#{additional_url}"
+        else
+          "#{u.scheme}://#{u.host}/#{u.path}/#{additional_url}"
+        end
     else
       additional_url
     end

data/spider.gemspec

@@ -14,5 +14,5 @@ spec = Gem::Specification.new do |s|
 A Web spidering library: handles robots.txt, scraping, finding more
 links, and doing it all over again.
 EOF
-  s.version = '0.4.2'
+  s.version = '0.4.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: spider
 version: !ruby/object:Gem::Version 
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors: 
 - John Nagro
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-07-06 00:00:00 -04:00
+date: 2008-10-09 00:00:00 -04:00
 default_executable: 
 dependencies: []