rawler 0.1.5 → 0.1.6

data/README.md

@@ -13,6 +13,8 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
         --password, -p <s>:   HTT Basic Password
             --wait, -w <f>:   Seconds to wait between requests, may be fractional e.g. '1.5' (default: 3.0)
                  --log, -l:   Log results to file rawler_log.txt
+         --logfile, -o <s>:   Specify logfile, implies --log (default: rawler_log.txt)
+                 --css, -c:   Check CSS links
              --version, -v:   Print version and exit
                 --help, -h:   Show this message
 

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.1.6

data/bin/rawler

@@ -17,7 +17,8 @@ EOS
   opt :username, "HTT Basic Username", :type => :string
   opt :password, "HTT Basic Password", :type => :string
   opt :wait, "Seconds to wait between requests, may be fractional e.g. '1.5'", :type => :float, :default => 3.0
-  opt :log, "Log results to file rawler_log.txt", :type => :boolean, :default => false
+  opt :log, "Log results to file #{Rawler::Base::DEFAULT_LOGFILE}", :type => :boolean, :default => false
+  opt :logfile, "Specify logfile, implies --log", :type => :string, :default => Rawler::Base::DEFAULT_LOGFILE
   opt :css, "Check CSS links", :type => :boolean, :default => false
 end
 

data/lib/rawler.rb

@@ -10,7 +10,7 @@ module Rawler
   mattr_accessor :url
   mattr_accessor :wait
   mattr_accessor :username, :password
-  mattr_accessor :log
+  mattr_accessor :log, :logfile
   mattr_accessor :css
 
   autoload :Base, "rawler/base"

data/lib/rawler/base.rb

@@ -1,6 +1,8 @@
 module Rawler
   class Base
 
+    DEFAULT_LOGFILE = "rawler_log.txt"
+
     attr_accessor :responses
 
     def initialize(url, output, options={})
@@ -12,9 +14,13 @@ module Rawler
       Rawler.username = options[:username]
       Rawler.password = options[:password]
       Rawler.wait     = options[:wait]
-      Rawler.log      = options[:log]
       Rawler.css      = options[:css]
-      @logfile = File.new("rawler_log.txt", "w") if Rawler.log
+
+      # Using a custom logfile implies logging.
+      Rawler.logfile  = options[:logfile] || DEFAULT_LOGFILE
+      Rawler.log      = options[:log] || Rawler.logfile != DEFAULT_LOGFILE
+
+      @logfile = File.new(Rawler.logfile, "w") if Rawler.log
     end
 
     def validate
@@ -55,9 +61,11 @@ module Rawler
     def add_status_code(link, from_url)
       response = Rawler::Request.get(link)
 
-      validate_page(response['Location'], from_url) if response['Location']
       record_response(response.code, link, from_url, response['Location'])
       responses[link] = { :status => response.code.to_i }
+
+      validate_page(response['Location'], from_url) if response['Location']
+
     rescue Errno::ECONNREFUSED
       error("Connection refused - #{link} - Called from: #{from_url}")
     rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT,

data/lib/rawler/crawler.rb

@@ -55,7 +55,7 @@ module Rawler
       else
         URI.parse(url).merge(path).to_s
       end
-    rescue URI::InvalidURIError
+    rescue URI::InvalidURIError, URI::InvalidComponentError
       write("Invalid url: #{path} - Called from: #{url}")
       nil
     end

data/rawler.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "rawler"
-  s.version = "0.1.5"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Oscar Del Ben"]
-  s.date = "2012-04-14"
+  s.date = "2012-07-17"
   s.description = "Rawler is a tool that crawls the links of your website"
   s.email = "info@oscardelben.com"
   s.executables = ["rawler"]
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/oscardelben/rawler"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.21"
+  s.rubygems_version = "1.8.23"
   s.summary = "Rawler is a tool that crawls the links of your website"
 
   if s.respond_to? :specification_version then

data/spec/lib/rawler/crawler_spec.rb

@@ -306,5 +306,20 @@ describe Rawler::Crawler do
       crawler.links.should == []
     end
   end
-  
+
+  context "invalid mailto" do
+    let(:content) { '<a href="mailto:obfuscated(at)example(dot)com">foo</a>' }
+    let(:url)     { 'http://example.com' }
+    let(:crawler) { Rawler::Crawler.new(url) }
+
+    before(:each) do
+      register(url, content)
+    end
+
+    it "should notify about the invalid url" do
+      output.should_receive(:error).with('Invalid url: mailto:obfuscated(at)example(dot)com - Called from: http://example.com')
+      crawler.links.should == []
+    end
+  end
+
 end

data/spec/lib/rawler_spec.rb

@@ -12,6 +12,40 @@ describe Rawler::Base do
     register('http://example.com', site)
   end
 
+  describe "logfile" do
+    it "should have default value" do
+      url = 'http://example.com'
+
+      Rawler::Base.new(url, output)
+      Rawler.logfile.should == Rawler::Base::DEFAULT_LOGFILE
+    end
+
+    it "should honor logfile option" do
+      url = 'http://example.com'
+      logfile = 'custom_logfile'.freeze
+
+      Rawler::Base.new(url, output, :logfile => logfile)
+      Rawler.logfile.should == logfile
+    end
+  end
+
+  describe "log" do
+    it "should be turned off by default" do
+      url = 'http://example.com'
+
+      Rawler::Base.new(url, output)
+      Rawler.log.should == false
+    end
+
+    it "should be turned on when assigning custom logfile" do
+      url = 'http://example.com'
+      logfile = 'custom_logfile'
+
+      Rawler::Base.new(url, output, :logfile => logfile)
+      Rawler.log.should == true
+    end
+  end
+
   describe "url encoding" do
     it "should encode url" do
       original = 'http://example.com/写程序容易出现的几个不好的地方'
@@ -95,6 +129,15 @@ describe Rawler::Base do
       rawler.validate
     end
 
+    it "should handle circular redirections" do
+      register('http://example.com', '<a href="/foo">foo</a>')
+      register('http://example.com/foo', '', 301, :location => 'http://example.com/foo')
+
+      output.should_receive(:warn).with('301 - http://example.com/foo - Called from: http://example.com - Following redirection to: http://example.com/foo')
+
+      rawler.validate
+    end
+
   end
   
   describe "get_status_code" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rawler
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-14 00:00:00.000000000 Z
+date: 2012-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -170,7 +170,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2522129833142198431
+      hash: -2845352824216855200
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Rawler is a tool that crawls the links of your website