rawler 0.0.7 → 0.0.8

data/lib/rawler/base.rb

@@ -21,29 +21,29 @@ module Rawler
     
     def validate_links_in_page(current_url)
       Rawler::Crawler.new(current_url).links.each do |page_url|
-        validate_page(page_url)
+        validate_page(page_url, current_url)
         # Todo: include this in a configuration option
         sleep(3)
       end
     end
     
-    def validate_page(page_url)
+    def validate_page(page_url, from_url)
       if not_yet_parsed?(page_url)
-        add_status_code(page_url) 
+        add_status_code(page_url, from_url) 
         validate_links_in_page(page_url) if same_domain?(page_url)
       end
     end
     
-    def add_status_code(link)
+    def add_status_code(link, from_url)
       response = Rawler::Request.get(link)
       
-      record_response(response.code, link)
+      record_response(response.code, link, from_url)
       responses[link] = { :status => response.code.to_i }
     rescue Errno::ECONNREFUSED
-      Rawler.output.error("Connection refused - '#{link}'")
+      Rawler.output.error("Connection refused - #{link} - Called from: #{from_url}")
     rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT,
-      EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError
-      Rawler.output.error("Connection problems - '#{link}'")
+      EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError
+      Rawler.output.error("Connection problems - #{link} - Called from: #{from_url}")
     end
     
     def same_domain?(link)
@@ -60,8 +60,13 @@ module Rawler
       Rawler.output.error(message)
     end
     
-    def record_response(code, link)
+    def record_response(code, link, from_url)
       message = "#{code} - #{link}"
+
+      if code.to_i >= 300
+        message += " - Called from: #{from_url}"
+      end
+
       code = code.to_i
       case code / 100
       when 1

data/lib/rawler/crawler.rb

@@ -4,6 +4,8 @@ module Rawler
     
     attr_accessor :url, :links
 
+    SKIP_FORMATS = /^(javascript|mailto)/
+
     def initialize(url)
       @url = url.strip
     end
@@ -59,7 +61,7 @@ module Rawler
       if ['http', 'https'].include?(scheme)
         true
       else
-        write("Invalid url - #{url}")
+        write("Invalid url - #{url}") unless url =~ SKIP_FORMATS
         false
       end
 

data/lib/rawler.rb

@@ -7,7 +7,7 @@ require 'logger'
 require 'rawler/core_extensions'
 
 module Rawler
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
   
   mattr_accessor :output
   mattr_accessor :url

data/spec/lib/rawler/crawler_spec.rb

@@ -100,26 +100,48 @@ describe Rawler::Crawler do
   end
   
   context "invalid urls" do
-    let(:url)     { 'http://example.com/path' }
-    let(:crawler) { Rawler::Crawler.new(url) }
-    let(:js_url)  { "javascript:fn('nbjmup;jhfs.esf{fio/dpn');" }
-    let(:content) { "<a href=\"#{js_url}\">foo</a><a name=\"foo\">" }
+
+    context "javascript" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:js_url)  { "javascript:fn('nbjmup;jhfs.esf{fio/dpn');" }
+      let(:content) { "<a href=\"#{js_url}\">foo</a><a name=\"foo\">" }
     
-    before(:each) do
-      register(url, content)
-    end
+      before(:each) do
+        register(url, content)
+      end
     
-    it "should parse relative links" do
-      crawler.links.should == []
+      it "should return empty links" do
+        crawler.links.should == []
+      end
+
+      it "should not report the error" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
     end
 
-    it "should report the error" do
-      crawler.should_receive(:write).with("Invalid url - javascript:fn('nbjmup;jhfs.esf%7Bfio/dpn');")
-      crawler.links
+    context "mailto" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"mailto:example@example.com\">foo</a><a name=\"foo\">" }
+    
+      before(:each) do
+        register(url, content)
+      end
+    
+      it "should return empty links" do
+        crawler.links.should == []
+      end
+
+      it "should not report the error" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
     end
+
   end
-  
-  
+
   context "content type" do
       
     ['text/plain', 'text/css', 'image/jpeg'].each do |content_type|

data/spec/lib/rawler_spec.rb

@@ -58,29 +58,31 @@ describe Rawler::Base do
       output.should_receive(:info).with('200 - http://example.com/foo1')
       output.should_receive(:info).with('200 - http://example.com/foo2')
       output.should_receive(:info).with('200 - http://external.com')
-      output.should_receive(:warn).with('302 - http://external.com/foo')
+      output.should_receive(:warn).with('302 - http://external.com/foo - Called from: http://example.com/foo1')
       
       rawler.validate
     end
-                
+
   end
   
   describe "get_status_code" do
 
     it "should add to 200 links" do
       url = 'http://example.com/foo'
+      from = 'http://other.com'
       register(url, '', 200)
       
-      rawler.send(:add_status_code, url)
+      rawler.send(:add_status_code, url, from)
       
       rawler.responses[url][:status].should == 200
     end
     
     it "should add to 302 links" do
       url = 'http://example.com/foo'
+      from = 'http://other.com'
       register(url, '', 302)
       
-      rawler.send(:add_status_code, url)
+      rawler.send(:add_status_code, url, from)
       
       rawler.responses[url][:status].should == 302
     end
@@ -94,24 +96,26 @@ describe Rawler::Base do
     
     it "should rescue from Errno::ECONNREFUSED" do
       url = 'http://example.com'
+      from = 'http://other.com'
       
       Rawler::Request.should_receive(:get).and_raise Errno::ECONNREFUSED
       
-      output.should_receive(:error).with("Connection refused - '#{url}'")
+      output.should_receive(:error).with("Connection refused - #{url} - Called from: #{from}")
       
-      rawler.send(:add_status_code, url)
+      rawler.send(:add_status_code, url, from)
     end
     
     [Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, EOFError,
-    Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError].each do |error|
+    Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError].each do |error|
        it "should rescue from #{error}" do
          url = 'http://example.com'
+         from = 'http://other.com'
 
          Rawler::Request.should_receive(:get).and_raise error
 
-         output.should_receive(:error).with("Connection problems - '#{url}'")
+         output.should_receive(:error).with("Connection problems - #{url} - Called from: #{from}")
 
-         rawler.send(:add_status_code, url)
+         rawler.send(:add_status_code, url, from)
        end   
     end
     
@@ -119,14 +123,15 @@ describe Rawler::Base do
   
   describe "record_response" do
     
-    let(:message) { 'foo' }
+    let(:link) { 'http://foo.com' }
+    let(:from) { 'http://bar.com' }
     
     context "response code 100" do
       %w!100, 150, 199!.each do |code|
 
         it "logger should receive info" do
-          output.should_receive(:info).with("#{code} - #{message}")
-          rawler.send(:record_response, code, message)
+          output.should_receive(:info).with("#{code} - #{link}")
+          rawler.send(:record_response, code, link, from)
         end
         
       end
@@ -136,8 +141,8 @@ describe Rawler::Base do
       %w!200, 250, 299!.each do |code|
 
         it "logger should receive info" do
-          output.should_receive(:info).with("#{code} - #{message}")
-          rawler.send(:record_response, code, message)
+          output.should_receive(:info).with("#{code} - #{link}")
+          rawler.send(:record_response, code, link, from)
         end
         
       end
@@ -147,8 +152,8 @@ describe Rawler::Base do
       %w!300, 350, 399!.each do |code|
 
         it "logger should receive warn" do
-          output.should_receive(:warn).with("#{code} - #{message}")
-          rawler.send(:record_response, code, message)
+          output.should_receive(:warn).with("#{code} - #{link} - Called from: #{from}")
+          rawler.send(:record_response, code, link, from)
         end
         
       end
@@ -158,8 +163,8 @@ describe Rawler::Base do
       %w!400, 450, 499!.each do |code|
 
         it "logger should receive info" do
-          output.should_receive(:error).with("#{code} - #{message}")
-          rawler.send(:record_response, code, message)
+          output.should_receive(:error).with("#{code} - #{link} - Called from: #{from}")
+          rawler.send(:record_response, code, link, from)
         end
         
       end
@@ -169,8 +174,8 @@ describe Rawler::Base do
       %w!400, 550, 599!.each do |code|
 
         it "logger should receive info" do
-          output.should_receive(:error).with("#{code} - #{message}")
-          rawler.send(:record_response, code, message)
+          output.should_receive(:error).with("#{code} - #{link} - Called from: #{from}")
+          rawler.send(:record_response, code, link, from)
         end
         
       end
@@ -180,8 +185,8 @@ describe Rawler::Base do
       let(:code) { 600 }
       
       it "logger should receive eror" do
-        output.should_receive(:error).with("Unknown code #{code} - #{message}")
-        rawler.send(:record_response, code, message)
+        output.should_receive(:error).with("Unknown code #{code} - #{link} - Called from: #{from}")
+        rawler.send(:record_response, code, link, from)
       end
     end
     

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 7
-  version: 0.0.7
+  - 8
+  version: 0.0.8
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-07 00:00:00 +01:00
+date: 2011-03-17 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency