rawler 0.0.1 → 0.0.2

data/History.txt

@@ -1,4 +1,10 @@
-=== 1.0.0 / 2011-01-10
+=== 0.0.2 / 2011-01-10
+
+* 1 major enhancement
+
+  * Handle relative urls
+
+=== 0.0.1 / 2011-01-10
 
 * 1 major enhancement
 

data/lib/rawler.rb

@@ -6,7 +6,7 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Rawler
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
   
   autoload :Base, "rawler/base"
   autoload :Crawler, "rawler/crawler"

data/lib/rawler/crawler.rb

@@ -12,11 +12,17 @@ module Rawler
       content = Net::HTTP.get(URI.parse(url))
       
       doc = Nokogiri::HTML(content)
-      doc.css('a').map { |a| a['href'] }
+      doc.css('a').map { |a| absolute_url(a['href']) }
     rescue Errno::ECONNREFUSED
       $output.puts "Couldn't connect to #{url}"
       []
     end
+    
+    private
+    
+    def absolute_url(path)
+      URI.parse(url).merge(path.to_s).to_s
+    end
   
   end
   

data/spec/unit/crawler_spec.rb

@@ -3,12 +3,40 @@ require File.dirname(__FILE__) + '/../spec_helper.rb'
 describe Rawler::Crawler do
   
   it "should parse all links" do
-    url = 'http://example.com'
+    url = 'http://example.com/'
     register(url, site)
     
     Rawler::Crawler.new(url).links.should == ['http://example.com/foo', 'http://external.com/bar']
   end
   
+  it "should return an empty array when raising Errno::ECONNREFUSED" do
+    url = 'http://example.com'
+    register(url, site)
+    
+    Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+    
+    crawler = Rawler::Crawler.new(url).links.should == []
+  end
+  
+  it "should parse relative links" do
+    url = 'http://example.com/path'
+    register(url, '<a href="/foo">foo</a>')
+    
+    Rawler::Crawler.new(url).links.should == ['http://example.com/foo']
+  end
+  
+  # it "should print a message when raising Errno::ECONNREFUSED" do
+  #   pending "refactor output. Don't use a global variable"
+  #   url = 'http://example.com'
+  #   register(url, site)
+  #   
+  #   Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+  #   
+  #   $stdout.should_receive(:puts).with("Couldn't connect to #{url}")
+  #   
+  #   Rawler::Crawler.new(url).links
+  # end
+  
   private
   
   def site

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rawler
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Oscar Del Ben