rawler 0.0.9 → 0.1.0

data/Manifest.txt

@@ -1,6 +1,5 @@
 Gemfile
 Gemfile.lock
-History.txt
 Manifest.txt
 README.txt
 Rakefile
@@ -19,4 +18,4 @@ spec/spec.opts
 spec/spec_helper.rb
 specs.watchr
 tasks/rspec.rake
-vendor/lib-trollop.rb
+vendor/lib-trollop.rb

data/README.txt

@@ -15,6 +15,7 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
 	where [options] are:
 	  --username, -u <s>:   HTTP Basic Username
 	  --password, -p <s>:   HTTP Basic Password
+                --wait:   Number of seconds to wait betweet requests (default 3)
 	       --version, -v:   Print version and exit
 	          --help, -h:   Show this message
 
@@ -46,6 +47,7 @@ And add them to the Manifest file.
 * Hugh Sasse
 * Ken Egozi
 * Robert Glaser
+* Stefan Schüßler
 * Vesa Vänskä
 
 See also https://github.com/oscardelben/rawler/contributors

data/bin/rawler

@@ -14,13 +14,12 @@ Usage:
 where [options] are:
 EOS
   
-  # opt :domain, "domain that you want to test", :type => :string
   opt :username, "HTT Basic Username", :type => :string
   opt :password, "HTT Basic Password", :type => :string
+  opt :wait, "Number of seconds to wait betweet requests", :default => 3
 end
 
-# Use dup to unfrozen string
-domain = ARGV.shift.dup
+domain = ARGV.shift
 
 if domain.nil?
   Trollop::die "Domain name is mandatory. Type --help for help"
@@ -30,5 +29,5 @@ else
   end
 end
 
-Rawler::Base.new(domain, $stdout, opts[:username], opts[:password]).validate
+Rawler::Base.new(domain, $stdout, opts).validate
 

data/lib/rawler.rb

@@ -7,10 +7,11 @@ require 'logger'
 require 'rawler/core_extensions'
 
 module Rawler
-  VERSION = '0.0.9'
+  VERSION = '0.1.0'
   
   mattr_accessor :output
   mattr_accessor :url
+  mattr_accessor :wait
   
   mattr_accessor :username, :password
   

data/lib/rawler/base.rb

@@ -4,13 +4,14 @@ module Rawler
     
     attr_accessor :responses
     
-    def initialize(url, output, username=nil, password=nil)
+    def initialize(url, output, options={})
       @responses = {}
 
       Rawler.url      = URI.escape(url)
       Rawler.output   = Logger.new(output)
-      Rawler.username = username
-      Rawler.password = password
+      Rawler.username = options[:username]
+      Rawler.password = options[:password]
+      Rawler.wait     = options[:wait].to_i
     end
     
     def validate
@@ -22,8 +23,7 @@ module Rawler
     def validate_links_in_page(current_url)
       Rawler::Crawler.new(current_url).links.each do |page_url|
         validate_page(page_url, current_url)
-        # Todo: include this in a configuration option
-        sleep(3)
+        sleep(Rawler.wait)
       end
     end
     

data/lib/rawler/crawler.rb

@@ -30,13 +30,11 @@ module Rawler
     private
     
     def absolute_url(path)
-      path = URI.encode(path.strip)
-      if path[0].chr == '/'
-        URI.parse(url).merge(path.to_s).to_s
-      elsif URI.parse(path).scheme.nil?
-        URI.parse(url).merge("/#{path.to_s}").to_s
-      else
+      path = URI.encode(path.strip, Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}#]"))
+      if URI.parse(path).scheme
         path
+      else
+        URI.parse(url).merge(path).to_s
       end
     rescue URI::InvalidURIError
       write("Invalid url: #{path} - Called from: #{url}")

data/spec/lib/rawler/crawler_spec.rb

@@ -36,16 +36,36 @@ describe Rawler::Crawler do
   
   context "relative paths" do
     
-    let(:url)     { 'http://example.com/path' }
-    let(:crawler) { Rawler::Crawler.new(url) }
-    let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a>' }
-    
-    before(:each) do
-      register(url, content)
+    context "base URL ends with a slash" do
+      
+      let(:url)     { 'http://example.com/dir1/dir2/' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a> <a href="../baz">baz</a>' }
+      
+      before(:each) do
+        register(url, content)
+      end
+      
+      it "should parse relative links" do
+        crawler.links.should == ['http://example.com/foo', 'http://example.com/dir1/dir2/bar', 'http://example.com/dir1/baz']
+      end
+      
     end
     
-    it "should parse relative links" do
-      crawler.links.should == ['http://example.com/foo', 'http://example.com/bar']
+    context "base URL doesn't end with a slash" do
+      
+      let(:url)     { 'http://example.com/dir1/dir2' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a> <a href="../baz">baz</a>' }
+      
+      before(:each) do
+        register(url, content)
+      end
+      
+      it "should parse relative links" do
+        crawler.links.should == ['http://example.com/foo', 'http://example.com/dir1/bar', 'http://example.com/baz']
+      end
+      
     end
     
   end
@@ -77,8 +97,8 @@ describe Rawler::Crawler do
       register(url, content)
     end
     
-    it "should parse urls with hashtags" do
-      crawler.links.should == ['http://example.com/foo%23bar']
+    it "should not encode hashtags" do
+      crawler.links.should == ['http://example.com/foo#bar']
     end
     
   end

data/spec/lib/rawler_spec.rb

@@ -99,12 +99,18 @@ describe Rawler::Base do
     end
     
     it "should save username and password" do
-      rawler = Rawler::Base.new('http://example.com', output, 'my_user', 'secret')
+      rawler = Rawler::Base.new('http://example.com', output, {:username => 'my_user', :password => 'secret'})
       
       Rawler.username.should == 'my_user'
       Rawler.password.should == 'secret'
     end
     
+    it "should save wait" do
+      rawler = Rawler::Base.new('http://example.com', output, {:wait => 5})
+      
+      Rawler.wait.should == 5
+    end
+    
     it "should rescue from Errno::ECONNREFUSED" do
       url = 'http://example.com'
       from = 'http://other.com'

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 9
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-25 00:00:00 +01:00
+date: 2011-03-29 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -71,13 +71,11 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
-- History.txt
 - Manifest.txt
 - README.txt
 files: 
 - Gemfile
 - Gemfile.lock
-- History.txt
 - Manifest.txt
 - README.txt
 - Rakefile

data/History.txt

@@ -1,12 +0,0 @@
-=== 0.0.2 / 2011-01-10
-
-* 1 major enhancement
-
-  * Handle relative urls
-
-=== 0.0.1 / 2011-01-10
-
-* 1 major enhancement
-
-  * Birthday!
-