rawler 0.1.7 → 0.1.8

data/.rspec

@@ -0,0 +1 @@
+--order rand

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+script: bundle exec rspec spec

data/Gemfile

@@ -5,13 +5,12 @@ source "http://rubygems.org"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
-gem 'nokogiri'
+gem 'nokogiri', '~> 1.5'
 
 group :development, :test do
   gem 'fakeweb'
   gem "rspec"
   gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
+  gem "bundler", "~> 1.0"
   gem "jeweler", "~> 1.6.4"
-  gem "rcov", ">= 0"
 end

data/Gemfile.lock

@@ -1,34 +1,52 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    diff-lcs (1.1.3)
+    activesupport (4.0.0)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    atomic (1.1.13)
+    diff-lcs (1.2.4)
     fakeweb (1.3.0)
-    git (1.2.5)
+    git (1.2.6)
+    i18n (0.6.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    nokogiri (1.5.0)
-    rake (0.9.2.2)
-    rcov (0.9.11)
-    rspec (2.7.0)
-      rspec-core (~> 2.7.0)
-      rspec-expectations (~> 2.7.0)
-      rspec-mocks (~> 2.7.0)
-    rspec-core (2.7.1)
-    rspec-expectations (2.7.0)
-      diff-lcs (~> 1.1.2)
-    rspec-mocks (2.7.0)
-    shoulda (2.11.3)
+    mini_portile (0.5.1)
+    minitest (4.7.5)
+    multi_json (1.7.9)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.2)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.3)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.1.5)
+    shoulda-matchers (2.3.0)
+      activesupport (>= 3.0.0)
+    thread_safe (0.1.2)
+      atomic
+    tzinfo (0.3.37)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.0.0)
+  bundler (~> 1.0)
   fakeweb
   jeweler (~> 1.6.4)
-  nokogiri
-  rcov
+  nokogiri (~> 1.5)
   rspec
   shoulda

data/README.md

@@ -9,14 +9,17 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
   rawler http://example.com [options]
 
       where [options] are:
-        --username, -u <s>:   HTT Basic Username
-        --password, -p <s>:   HTT Basic Password
+        --username, -u <s>:   HTTP Basic Username
+        --password, -p <s>:   HTTP Basic Password
             --wait, -w <f>:   Seconds to wait between requests, may be fractional e.g. '1.5' (default: 3.0)
                  --log, -l:   Log results to file rawler_log.txt
          --logfile, -o <s>:   Specify logfile, implies --log (default: rawler_log.txt)
                  --css, -c:   Check CSS links
-            --skip, -s <s>:   Skip URLS that match a regexp
-           --iskip, -i <s>:   Skip URLS that match a case insensitive regexp
+            --skip, -s <s>:   Skip URLs that match a regexp
+           --iskip, -i <s>:   Skip URLs that match a case insensitive regexp
+             --include <s>:   Only include URLs that match a regexp
+            --iinclude <s>:   Only include URLs that match a case insensitive regexp
+               --local <s>:   Restrict to the given URL and below. Equivalent to '--include ^http://mysite.com/*'.
              --version, -v:   Print version and exit
                 --help, -h:   Show this message
 

data/Rakefile

@@ -28,24 +28,7 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
-end
-
-task :default => :test
-
-require 'rake/rdoctask'
+require 'rdoc/task'
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
@@ -54,16 +37,3 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
-
-desc 'generate docs'
-task :rocco do
-  #%x!rm -r html/*!
-
-  Dir.chdir "lib"
-
-  files = Dir['**/*.*']
-  
-  files.each do |file|
-    %x!rocco #{file} -o ../html!
-  end
-end

data/VERSION

@@ -1 +1 @@
-0.1.7
+0.1.8

data/bin/rawler

@@ -14,14 +14,17 @@ Usage:
 where [options] are:
 EOS
   
-  opt :username, "HTT Basic Username", :type => :string
-  opt :password, "HTT Basic Password", :type => :string
+  opt :username, "HTTP Basic Username", :type => :string
+  opt :password, "HTTP Basic Password", :type => :string
   opt :wait, "Seconds to wait between requests, may be fractional e.g. '1.5'", :type => :float, :default => 3.0
   opt :log, "Log results to file #{Rawler::Base::DEFAULT_LOGFILE}", :type => :boolean, :default => false
   opt :logfile, "Specify logfile, implies --log", :type => :string, :default => Rawler::Base::DEFAULT_LOGFILE
   opt :css, "Check CSS links", :type => :boolean, :default => false
-  opt :skip, "Skip URLS that match a pattern", :type => :string
-  opt :iskip, "Skip URLS that match a case insensitive pattern", :type => :string
+  opt :skip, "Skip URLs that match a pattern", :type => :string
+  opt :iskip, "Skip URLs that match a case insensitive pattern", :type => :string
+  opt :include, "Only include URLS that match a pattern", :type => :string
+  opt :iinclude, "Only include URLS that match a case insensitive pattern. Equivalent to '--include ^http://mysite.com/*'.", :type => :string
+  opt :local, "Restrict to the given URL and below", :type => :boolean, :default => false
 end
 
 

data/lib/rawler.rb

@@ -12,6 +12,7 @@ module Rawler
   mattr_accessor :username, :password
   mattr_accessor :log, :logfile
   mattr_accessor :css
+  mattr_accessor :include_url_pattern
   mattr_accessor :skip_url_pattern
 
   autoload :Base, "rawler/base"
@@ -28,7 +29,20 @@ module Rawler
     @@url = url
   end
 
-  def self.set_skip_pattern(pattern, icase)
-    self.skip_url_pattern = pattern.nil? ? nil : Regexp.new(pattern, icase ? Regexp::IGNORECASE : nil )
+  def self.create_regex(pattern, icase=false)
+    pattern.nil? ? nil : Regexp.new(pattern, icase ? Regexp::IGNORECASE : nil )
+  end
+
+  def self.set_include_pattern(pattern, icase=false)
+    self.include_url_pattern = self.create_regex(pattern, icase)
+  end
+
+  def self.set_skip_pattern(pattern, icase=false)
+    self.skip_url_pattern = self.create_regex(pattern, icase)
+  end
+
+  def self.local=(is_local)
+    pattern = is_local ? "^#{self.url}" : nil
+    self.set_include_pattern(pattern)
   end
 end

data/lib/rawler/base.rb

@@ -15,6 +15,11 @@ module Rawler
       Rawler.password = options[:password]
       Rawler.wait     = options[:wait]
       Rawler.css      = options[:css]
+      
+      Rawler.local    = options[:local]
+
+      Rawler.set_include_pattern(options[:include], false) unless options[:include].nil?
+      Rawler.set_include_pattern(options[:iinclude], true) unless options[:iinclude].nil?
 
       Rawler.set_skip_pattern(options[:skip], false) unless options[:skip].nil?
       Rawler.set_skip_pattern(options[:iskip], true) unless options[:iskip].nil?

data/lib/rawler/crawler.rb

@@ -91,6 +91,8 @@ module Rawler
       scheme = URI.parse(url).scheme
       if url =~ Rawler.skip_url_pattern
         false
+      elsif Rawler.include_url_pattern && url !~ Rawler.include_url_pattern
+        false
       elsif ['http', 'https'].include?(scheme)
         true
       else

data/rawler.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "rawler"
-  s.version = "0.1.7"
+  s.version = "0.1.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Oscar Del Ben"]
-  s.date = "2012-12-02"
+  s.date = "2013-09-07"
   s.description = "Rawler is a tool that crawls the links of your website"
   s.email = "info@oscardelben.com"
   s.executables = ["rawler"]
@@ -18,6 +18,8 @@ Gem::Specification.new do |s|
     "README.md"
   ]
   s.files = [
+    ".rspec",
+    ".travis.yml",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
@@ -39,8 +41,6 @@ Gem::Specification.new do |s|
     "spec/lib/rawler_spec.rb",
     "spec/spec.opts",
     "spec/spec_helper.rb",
-    "test/helper.rb",
-    "test/test_rawler.rb",
     "vendor/lib-trollop.rb"
   ]
   s.homepage = "http://github.com/oscardelben/rawler"
@@ -53,30 +53,27 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, ["~> 1.5"])
       s.add_development_dependency(%q<fakeweb>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
     else
-      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, ["~> 1.5"])
       s.add_dependency(%q<fakeweb>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
-    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, ["~> 1.5"])
     s.add_dependency(%q<fakeweb>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
   end
 end
 

data/spec/lib/rawler/crawler_spec.rb

@@ -8,12 +8,12 @@ describe Rawler::Crawler do
   let(:output)  { double('output', :error => nil) }
 
   before(:each) do
-    Rawler.stub!(:url).and_return(url)
-    Rawler.stub!(:output).and_return(output)
+    Rawler.stub(:url).and_return(url)
+    Rawler.stub(:output).and_return(output)
   end
 
   context "basic functionality" do
-    
+
     let(:url) { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:content) {
@@ -32,87 +32,87 @@ describe Rawler::Crawler do
     it "should parse all links" do
       crawler.links.should == ['http://example.com/foo', 'http://external.com/bar']
     end
-    
+
     it "should parse css links" do
       crawler.css_links.should == ['http://example.com/css/styles.css']
-    end 
+    end
   end
-  
+
   context "relative paths" do
-    
+
     context "base URL ends with a slash" do
-      
+
       let(:url)     { 'http://example.com/dir1/dir2/' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a> <a href="../baz">baz</a>' }
-      
+
       before(:each) do
         register(url, content)
       end
-      
+
       it "should parse relative links" do
         crawler.links.should == ['http://example.com/foo', 'http://example.com/dir1/dir2/bar', 'http://example.com/dir1/baz']
       end
-      
+
     end
-    
+
     context "base URL doesn't end with a slash" do
-      
+
       let(:url)     { 'http://example.com/dir1/dir2' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { '<a href="/foo">foo</a> <a href="bar">bar</a> <a href="../baz">baz</a>' }
-      
+
       before(:each) do
         register(url, content)
       end
-      
+
       it "should parse relative links" do
         crawler.links.should == ['http://example.com/foo', 'http://example.com/dir1/bar', 'http://example.com/baz']
       end
-      
+
     end
-    
+
   end
-  
+
   context "different domains" do
-    
+
     let(:url)     { 'http://external.com/path' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:content) { '<a href="/foo">foo</a>' }
-    
+
     before(:each) do
-      Rawler.stub!(:url).and_return('http://example.com')
+      Rawler.stub(:url).and_return('http://example.com')
       register(url, content)
     end
-    
+
     it "should parse relative links" do
       crawler.links.should == []
     end
-    
+
   end
-  
+
   context "urls with hash tags" do
-    
+
     let(:url)     { 'http://example.com/path' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:content) { '<a href="/foo#bar">foo</a>' }
-    
+
     before(:each) do
       register(url, content)
     end
-    
+
     it "should not encode hashtags" do
       crawler.links.should == ['http://example.com/foo#bar']
     end
-    
+
   end
-  
+
   context "urls with unicode characters" do
-    
+
     let(:url)     { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:content) { '<a href="http://example.com/写程序容易出现的几个不好的地方">foo</a>' }
-    
+
     before(:each) do
       register(url, content)
     end
@@ -120,9 +120,9 @@ describe Rawler::Crawler do
     it "should parse unicode links" do
       crawler.links.should == ['http://example.com/%E5%86%99%E7%A8%8B%E5%BA%8F%E5%AE%B9%E6%98%93%E5%87%BA%E7%8E%B0%E7%9A%84%E5%87%A0%E4%B8%AA%E4%B8%8D%E5%A5%BD%E7%9A%84%E5%9C%B0%E6%96%B9']
     end
-    
+
   end
-  
+
   context "invalid urls" do
 
     context "javascript" do
@@ -130,11 +130,11 @@ describe Rawler::Crawler do
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:js_url)  { "javascript:fn('nbjmup;jhfs.esf{fio/dpn');" }
       let(:content) { "<a href=\"#{js_url}\">foo</a><a name=\"foo\">" }
-    
+
       before(:each) do
         register(url, content)
       end
-    
+
       it "should return empty links" do
         crawler.links.should == []
       end
@@ -149,11 +149,11 @@ describe Rawler::Crawler do
       let(:url)     { 'http://example.com/path' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { "<a href=\"mailto:example@example.com\">foo</a><a name=\"foo\">" }
-    
+
       before(:each) do
         register(url, content)
       end
-    
+
       it "should return empty links" do
         crawler.links.should == []
       end
@@ -163,16 +163,16 @@ describe Rawler::Crawler do
         crawler.links
       end
     end
-    
+
     context "callto" do
       let(:url)     { 'http://example.com/path' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { "<a href=\"callto:home22\">foo</a><a name=\"foo\">" }
-    
+
       before(:each) do
         register(url, content)
       end
-    
+
       it "should return empty links" do
         crawler.links.should == []
       end
@@ -187,12 +187,12 @@ describe Rawler::Crawler do
       let(:url)     { 'http://example.com/path' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
-    
+
       before(:each) do
         Rawler.set_skip_pattern('\/search\/(.*\/)?page:[2-9]', false)
         register(url, content)
       end
-    
+
       it "should return one links" do
         crawler.links.length.should eql(1)
       end
@@ -201,63 +201,144 @@ describe Rawler::Crawler do
         crawler.should_not_receive(:write)
         crawler.links
       end
+
+      after(:each) do
+        Rawler.set_skip_pattern(nil)
+      end
     end
 
     context "case-insensitive skip matches" do
       let(:url)     { 'http://example.com/path' }
       let(:crawler) { Rawler::Crawler.new(url) }
       let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
-    
+
       before(:each) do
         Rawler.set_skip_pattern('\/seArcH\/(.*\/)?PAGE:[2-9]', true)
         register(url, content)
       end
-    
+
+      it "should return one links" do
+        crawler.links.length.should eql(1)
+      end
+
+      it "should not report that it's skipping" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+
+      after(:each) do
+        Rawler.set_skip_pattern(nil)
+      end
+    end
+
+    context "include matches" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
+
+      before(:each) do
+        Rawler.set_include_pattern('\/search\/(.*\/)?page:[2-9]', false)
+        register(url, content)
+      end
+
+      it "should return one links" do
+        crawler.links.length.should eql(1)
+        crawler.links.should eq(['http://example.com/search/page:2/'])
+      end
+
+      it "should not report that it's including" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+
+      after(:each) do
+        Rawler.set_include_pattern(nil)
+      end
+    end
+
+    context "case-insensitive include matches" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
+
+      before(:each) do
+        Rawler.set_include_pattern('\/seArcH\/(.*\/)?PAGE:[2-9]', true)
+        register(url, content)
+      end
+
       it "should return one links" do
         crawler.links.length.should eql(1)
       end
 
+      it "should not report that it's including" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+
+      after(:each) do
+        Rawler.set_include_pattern(nil)
+      end
+    end
+
+    context "non-local site should be omitted when local flag is used" do
+      let(:url)     { 'http://example.com/' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"http://example.com/page1/\">foo</a><a href=\"http://example.org/page2\">foo</a>" }
+
+      before(:each) do
+        Rawler.local = true
+        register(url, content)
+      end
+
+      it "should return one link" do
+        crawler.links.length.should eql(1)
+      end
+
       it "should not report that it's skipping" do
         crawler.should_not_receive(:write)
         crawler.links
       end
+
+      after(:each) do
+        Rawler.local = false
+      end
     end
 
   end
 
   context "content type" do
-      
+
     ['text/plain', 'text/css', 'image/jpeg'].each do |content_type|
-    
+
       let(:url)     { 'http://example.com' }
       let(:crawler) { Rawler::Crawler.new(url) }
-    
+
       before(:each) do
         register(url, '', 200, :content_type => content_type)
       end
-    
+
       it "should ignore '#{content_type}'" do
         crawler.links.should == []
       end
-  
+
     end
   end
-  
+
   context "Exceptions" do
-    
+
     let(:url)     { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
-    
+
     before(:each) do
       register(url, '')
     end
-    
+
     context "Errno::ECONNREFUSED" do
-      
+
       before(:each) do
-        Rawler::Request.stub!(:get).and_raise Errno::ECONNREFUSED
+        Rawler::Request.stub(:get).and_raise Errno::ECONNREFUSED
       end
-      
+
       it "should return an empty array" do
         crawler.links.should == []
       end
@@ -266,14 +347,14 @@ describe Rawler::Crawler do
         output.should_receive(:error).with("Couldn't connect to #{url}")
 
         crawler.links
-      end      
-   
+      end
+
     end
-    
+
     context "Errno::ETIMEDOUT" do
-      
+
       before(:each) do
-        Rawler::Request.stub!(:get).and_raise Errno::ETIMEDOUT
+        Rawler::Request.stub(:get).and_raise Errno::ETIMEDOUT
       end
 
       it "should return an empty array when raising Errno::ETIMEDOUT" do
@@ -285,33 +366,33 @@ describe Rawler::Crawler do
 
         crawler.links
       end
-  
+
     end
-    
+
   end
-  
+
   context "http basic" do
-    
+
     let(:url)     { 'http://example.com' }
     let(:content) { '<a href="http://example.com/secret-path">foo</a>' }
     let(:crawler) { Rawler::Crawler.new('http://example.com/secret') }
-    
+
     before(:each) do
       register('http://example.com/secret', '', :status => ["401", "Unauthorized"])
       register('http://foo:bar@example.com/secret', content)
 
-      Rawler.stub!(:username).and_return('foo')
-      Rawler.stub!(:password).and_return('bar')
+      Rawler.stub(:username).and_return('foo')
+      Rawler.stub(:password).and_return('bar')
     end
-   
+
     it "should crawl http basic pages" do
       crawler.links.should == ['http://example.com/secret-path']
     end
-    
+
   end
-  
+
   context "url domain" do
-    
+
     let(:content) {
       content = <<-content
         <a href="http://example.com/valid">foo</a>
@@ -322,11 +403,11 @@ describe Rawler::Crawler do
     }
     let(:url)     { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
-    
+
     before(:each) do
       register(url, content)
     end
-  
+
     it "should ignore links other than http or https" do
       crawler.links.should == ['http://example.com/valid', 'https://foo.com', 'http://fooo.com']
     end
@@ -336,11 +417,11 @@ describe Rawler::Crawler do
     let(:content) { '<a href="http://foo;bar">foo</a>' }
     let(:url)     { 'http://example.com' }
     let(:crawler) { Rawler::Crawler.new(url) }
-    
+
     before(:each) do
       register(url, content)
     end
-  
+
     it "should notify about the invalid url" do
       output.should_receive(:error).with('Invalid url: http://foo;bar - Called from: http://example.com')
       crawler.links.should == []

data/spec/lib/rawler_spec.rb

@@ -8,7 +8,7 @@ describe Rawler::Base do
   let(:rawler)  { Rawler::Base.new('http://example.com', output) }
   
   before(:each) do
-    Rawler.stub!(:output).and_return(output)
+    Rawler.stub(:output).and_return(output)
     register('http://example.com', site)
   end
 
@@ -94,6 +94,7 @@ describe Rawler::Base do
     end
     
     it "should not validate links on external pages" do
+      register('http://example.com/', '<a href="http://external.com/foo">x</a>')
       register('http://example.com/foo', '<a href="http://external.com/foo">x</a>')
       register('http://external.com/foo', '<a href="http://external.com/bar">x</a>')
       register('http://external.com/bar', '')

data/spec/spec_helper.rb

@@ -1,9 +1,9 @@
 module Kernel
-  
+
   def sleep(duration)
     nil
   end
-  
+
 end
 
 
@@ -16,3 +16,8 @@ FakeWeb.allow_net_connect = false
 def register(uri, content, status=200, options={})
   FakeWeb.register_uri(:any, uri, { :body => content, :status => status, :content_type => 'text/html' }.merge(options))
 end
+
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rawler
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-02 00:00:00.000000000 Z
+date: 2013-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
@@ -107,22 +107,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.6.4
-- !ruby/object:Gem::Dependency
-  name: rcov
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Rawler is a tool that crawls the links of your website
 email: info@oscardelben.com
 executables:
@@ -132,6 +116,8 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
+- .rspec
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -153,8 +139,6 @@ files:
 - spec/lib/rawler_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- test/helper.rb
-- test/test_rawler.rb
 - vendor/lib-trollop.rb
 homepage: http://github.com/oscardelben/rawler
 licenses:
@@ -171,7 +155,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -477710177479430630
+      hash: 3460091796500092184
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/test/helper.rb

@@ -1,18 +0,0 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'rawler'
-
-class Test::Unit::TestCase
-end

data/test/test_rawler.rb

@@ -1,7 +0,0 @@
-require 'helper'
-
-class TestRawler < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
-  end
-end