rawler 0.0.8 → 0.0.9

data/README.txt

@@ -13,8 +13,8 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
   rawler http://example.com [options]
 
 	where [options] are:
-	  --username, -u <s>:   HTT Basic Username
-	  --password, -p <s>:   HTT Basic Password
+	  --username, -u <s>:   HTTP Basic Username
+	  --password, -p <s>:   HTTP Basic Password
 	       --version, -v:   Print version and exit
 	          --help, -h:   Show this message
 
@@ -40,13 +40,6 @@ If you add files, run:
 
 And add them to the Manifest file.
 
-== TODO
-
-* Add logger levels
-* Follow redirects, but still inform about them
-* Respect robots.txt
-* Export to html
-
 == CONTRIBUTORS:
 
 * bcoob

data/lib/rawler.rb

@@ -7,7 +7,7 @@ require 'logger'
 require 'rawler/core_extensions'
 
 module Rawler
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
   
   mattr_accessor :output
   mattr_accessor :url

data/lib/rawler/base.rb

@@ -36,14 +36,16 @@ module Rawler
     
     def add_status_code(link, from_url)
       response = Rawler::Request.get(link)
+
+      validate_page(response['Location'], from_url) if response['Location']
       
-      record_response(response.code, link, from_url)
+      record_response(response.code, link, from_url, response['Location'])
       responses[link] = { :status => response.code.to_i }
     rescue Errno::ECONNREFUSED
-      Rawler.output.error("Connection refused - #{link} - Called from: #{from_url}")
+      error("Connection refused - #{link} - Called from: #{from_url}")
     rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT,
       EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError
-      Rawler.output.error("Connection problems - #{link} - Called from: #{from_url}")
+      error("Connection problems - #{link} - Called from: #{from_url}")
     end
     
     def same_domain?(link)
@@ -54,18 +56,18 @@ module Rawler
       responses[link].nil?
     end
     
-    def write(message)
-      # TODO: This may not always be an error message, 
-      # but that will make it show up most of the time
+    def error(message)
       Rawler.output.error(message)
     end
     
-    def record_response(code, link, from_url)
+    def record_response(code, link, from_url, redirection=nil)
       message = "#{code} - #{link}"
 
       if code.to_i >= 300
         message += " - Called from: #{from_url}"
       end
+          
+      message += " - Following redirection to: #{redirection}" if redirection
 
       code = code.to_i
       case code / 100

data/lib/rawler/crawler.rb

@@ -19,10 +19,10 @@ module Rawler
       
       doc = Nokogiri::HTML(response.body)
       doc.css('a').map { |a| a['href'] }.select { |url| !url.nil? }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
-    rescue Errno::ECONNREFUSED
+    rescue Errno::ECONNREFUSED # TODO: add called from
       write("Couldn't connect to #{url}")
       []
-    rescue Errno::ETIMEDOUT
+    rescue Errno::ETIMEDOUT # TODO: add called from
       write("Connection to #{url} timed out")
       []
     end
@@ -38,8 +38,12 @@ module Rawler
       else
         path
       end
+    rescue URI::InvalidURIError
+      write("Invalid url: #{path} - Called from: #{url}")
+      nil
     end
     
+    # TODO: add 'called from in a more pragmatic way as an optional parameter
     def write(message)
       Rawler.output.error(message)
     end

data/spec/lib/rawler/crawler_spec.rb

@@ -5,7 +5,7 @@ require File.dirname(__FILE__) + '/../../spec_helper.rb'
 describe Rawler::Crawler do
 
   let(:url)    { 'http://example.com' }
-  let(:output) { double("output", :error => nil) }
+  let(:output)  { double('output', :error => nil) }
 
   before(:each) do
     Rawler.stub!(:url).and_return(url)
@@ -164,11 +164,9 @@ describe Rawler::Crawler do
     
     let(:url)     { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
-    let(:output)  { double('output', :error => nil) }
     
     before(:each) do
       register(url, '')
-      Rawler.stub!(:output).and_return(output)
     end
     
     context "Errno::ECONNREFUSED" do
@@ -250,5 +248,20 @@ describe Rawler::Crawler do
       crawler.links.should == ['http://example.com/valid', 'https://foo.com', 'http://fooo.com']
     end
   end
+
+  context "invalid urls" do
+    let(:content) { '<a href="http://foo;bar">foo</a>' }
+    let(:url)     { 'http://example.com' }
+    let(:crawler) { Rawler::Crawler.new(url) }
+    
+    before(:each) do
+      register(url, content)
+    end
+  
+    it "should notify about the invalid url" do
+      output.should_receive(:error).with('Invalid url: http://foo;bar - Called from: http://example.com')
+      crawler.links.should == []
+    end
+  end
   
 end

data/spec/lib/rawler_spec.rb

@@ -53,16 +53,27 @@ describe Rawler::Base do
       register('http://example.com/foo1', '<a href="http://external.com/foo">x</a>')
       register('http://example.com/foo2', '')
       register('http://external.com', '')
-      register('http://external.com/foo', '', 302)
+      register('http://external.com/foo', '', 301)
       
       output.should_receive(:info).with('200 - http://example.com/foo1')
       output.should_receive(:info).with('200 - http://example.com/foo2')
       output.should_receive(:info).with('200 - http://external.com')
-      output.should_receive(:warn).with('302 - http://external.com/foo - Called from: http://example.com/foo1')
+      output.should_receive(:warn).with('301 - http://external.com/foo - Called from: http://example.com/foo1')
       
       rawler.validate
     end
 
+    it "should follow redirections but inform about them" do
+      register('http://example.com', '<a href="/foo">foo</a>')
+      register('http://example.com/foo', '', 301, :location => 'http://example.com/bar')
+      register('http://example.com/bar', '')
+
+      output.should_receive(:warn).with('301 - http://example.com/foo - Called from: http://example.com - Following redirection to: http://example.com/bar')
+      output.should_receive(:info).with('200 - http://example.com/bar')
+
+      rawler.validate
+    end
+
   end
   
   describe "get_status_code" do
@@ -77,14 +88,14 @@ describe Rawler::Base do
       rawler.responses[url][:status].should == 200
     end
     
-    it "should add to 302 links" do
+    it "should add to 301 links" do
       url = 'http://example.com/foo'
       from = 'http://other.com'
-      register(url, '', 302)
+      register(url, '', 301)
       
       rawler.send(:add_status_code, url, from)
       
-      rawler.responses[url][:status].should == 302
+      rawler.responses[url][:status].should == 301
     end
     
     it "should save username and password" do

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 8
-  version: 0.0.8
+  - 9
+  version: 0.0.9
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-17 00:00:00 +01:00
+date: 2011-03-25 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency