rawler 0.0.1

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2011-01-10
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,13 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/rawler
+lib/rawler/base.rb
+lib/rawler/crawler.rb
+lib/rawler.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/unit/base_spec.rb
+spec/unit/crawler_spec.rb
+tasks/rspec.rake

data/README.txt

@@ -0,0 +1,47 @@
+= rawler
+
+* http://github.com/#{github_username}/#{project_name}
+
+== DESCRIPTION:
+
+Rawler is a Ruby library that crawls your website and see the status code of each of your links. Useful for finding dead links.
+
+== SYNOPSIS:
+
+  rawler http://example.com
+
+== INSTALL:
+
+gem install rawler
+
+== TODO
+
+* Handle https (now returns 400). See http://stackoverflow.com/questions/1719809/ruby-on-rails-https-post-bad-request
+* Export to html
+* Handle multiple urls at once
+* Add user agent
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Oscar Del Ben
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,24 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+# Hoe.plugin :compiler
+# Hoe.plugin :gem_prelude_sucks
+# Hoe.plugin :inline
+# Hoe.plugin :racc
+# Hoe.plugin :rubyforge
+
+Hoe.spec 'rawler' do
+  # HEY! If you fill these out in ~/.hoe_template/Rakefile.erb then
+  # you'll never have to touch them again!
+  # (delete this comment too, of course)
+
+  developer('Oscar Del Ben', 'info@oscardelben.com')
+
+  self.rubyforge_name = 'oscardelben'
+  
+  extra_deps << ['nokogiri']
+end
+
+# vim: syntax=ruby

data/bin/rawler

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '..', '/lib/rawler.rb')
+
+domain = ARGV[0]
+
+if domain.nil?
+  puts "Usage: rawler http://example.com"
+end
+
+Rawler::Base.new(domain, $stdout).validate

data/lib/rawler.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+require 'net/http'
+require 'nokogiri'
+
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Rawler
+  VERSION = '0.0.1'
+  
+  autoload :Base, "rawler/base"
+  autoload :Crawler, "rawler/crawler"
+end

data/lib/rawler/base.rb

@@ -0,0 +1,61 @@
+module Rawler
+  
+  class Base
+    
+    attr_accessor :url, :responses
+    
+    def initialize(url, output)
+      @url = url
+      @responses = {}
+      $output = output
+    end
+    
+    def validate
+      validate_links_in_page(url)
+    end
+    
+    private
+    
+    def validate_links_in_page(current_url)
+      Rawler::Crawler.new(current_url).links.each do |page_url|
+        validate_page(page_url)
+      end
+    end
+    
+    def validate_page(page_url)
+      if not_yet_parsed?(page_url)
+        add_status_code(page_url) 
+        validate_links_in_page(page_url) if same_domain?(page_url)
+      end
+    end
+    
+    def add_status_code(link)
+      uri = URI.parse(link)
+      
+      response = nil
+
+      Net::HTTP.start(uri.host, uri.port) do |http|
+        path = (uri.path.size == 0)  ? "/" : uri.path
+        response = http.head(path, {'User-Agent'=>'Rawler'})
+      end
+      
+      $output.puts("#{response.code} - #{link}")
+      responses[link] = { :status => response.code.to_i }
+    rescue Errno::ECONNREFUSED
+      puts "Connection refused - '#{link}'"
+    rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+           Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError
+      puts "Connection problems - #{link}"
+    end
+    
+    def same_domain?(link)
+      URI.parse(url).host == URI.parse(link).host
+    end
+    
+    def not_yet_parsed?(link)
+      responses[link].nil?
+    end
+    
+  end
+  
+end

data/lib/rawler/crawler.rb

@@ -0,0 +1,23 @@
+module Rawler
+  
+  class Crawler
+    
+    attr_accessor :url, :links
+
+    def initialize(url)
+      @url = url
+    end
+    
+    def links
+      content = Net::HTTP.get(URI.parse(url))
+      
+      doc = Nokogiri::HTML(content)
+      doc.css('a').map { |a| a['href'] }
+    rescue Errno::ECONNREFUSED
+      $output.puts "Couldn't connect to #{url}"
+      []
+    end
+  
+  end
+  
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'rawler'
+require 'fakeweb'
+
+FakeWeb.allow_net_connect = false
+
+def register(uri, content, status=200)
+  FakeWeb.register_uri(:any, uri, :body => content, :status => status)
+end

data/spec/unit/base_spec.rb

@@ -0,0 +1,93 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+
+describe Rawler::Base do
+
+  let(:output)  { double('output').as_null_object }
+  let(:rawler)  { Rawler::Base.new('http://example.com', output) }
+  
+  before(:each) do
+    register('http://example.com', site)
+  end
+  
+  describe "validate_links" do
+    
+    it "should validate links recursively" do
+      register('http://example.com/foo1', '<a href="http://external.com/foo">x</a>')
+      register('http://example.com/foo2', '')
+      register('http://external.com', '')
+      register('http://external.com/foo', '')
+
+      rawler.validate
+      
+      rawler.responses['http://example.com/foo1'].should_not be_nil
+      rawler.responses['http://example.com/foo2'].should_not be_nil
+      rawler.responses['http://external.com'].should_not be_nil
+      rawler.responses['http://external.com/foo'].should_not be_nil
+    end
+    
+    it "should not validate links on external pages" do
+      register('http://example.com/foo', '<a href="http://external.com/foo">x</a>')
+      register('http://external.com/foo', '<a href="http://external.com/bar">x</a>')
+      register('http://external.com/bar', '')
+      
+      rawler.validate
+      
+      rawler.responses['http://external.com/foo'].should_not be_nil
+      rawler.responses['http://external.com/bar'].should be_nil
+    end
+    
+    it "should output results" do
+      register('http://example.com/foo1', '<a href="http://external.com/foo">x</a>')
+      register('http://example.com/foo2', '')
+      register('http://external.com', '')
+      register('http://external.com/foo', '', 302)
+      
+      output.should_receive(:puts).with('200 - http://example.com/foo1')
+      output.should_receive(:puts).with('200 - http://example.com/foo2')
+      output.should_receive(:puts).with('200 - http://external.com')
+      output.should_receive(:puts).with('302 - http://external.com/foo')
+      
+      rawler.validate
+    end
+            
+  end
+  
+  describe "get_status_code" do
+
+    it "should add to 200 links" do
+      url = 'http://example.com/foo'
+      register(url, '', 200)
+      
+      rawler.send(:add_status_code, url)
+      
+      rawler.responses[url][:status].should == 200
+    end
+    
+    it "should add to 302 links" do
+      url = 'http://example.com/foo'
+      register(url, '', 302)
+      
+      rawler.send(:add_status_code, url)
+      
+      rawler.responses[url][:status].should == 302
+    end
+    
+  end
+  
+  
+  private
+  
+  def site
+    <<-site
+      <html>
+        <body>
+          <a href="http://example.com/foo1">foo1</a>
+          <a href="http://example.com/foo2">foo2</a>
+
+          <a href="http://external.com">external</a>
+        </body>
+      </html>
+    site
+  end
+  
+end 

data/spec/unit/crawler_spec.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+
+describe Rawler::Crawler do
+  
+  it "should parse all links" do
+    url = 'http://example.com'
+    register(url, site)
+    
+    Rawler::Crawler.new(url).links.should == ['http://example.com/foo', 'http://external.com/bar']
+  end
+  
+  private
+  
+  def site
+    <<-site
+      <!DOCTYPE html>
+      <html>
+      	<body>
+      		<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+
+      		<p><a href="http://example.com/foo">foo</a></p>
+
+      		<p><a href="http://external.com/bar">bar</a></p>
+
+      	</body>
+      </html>
+    site
+  end
+  
+end

data/tasks/rspec.rake

@@ -0,0 +1,21 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems' unless ENV['NO_RUBYGEMS']
+  require 'spec'
+end
+begin
+  require 'spec/rake/spectask'
+rescue LoadError
+  puts <<-EOS
+To use rspec for testing you must install rspec gem:
+    gem install rspec
+EOS
+  exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification 
+name: rawler
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Oscar Del Ben
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-10 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 47
+        segments: 
+        - 2
+        - 8
+        - 0
+        version: 2.8.0
+  type: :development
+  version_requirements: *id002
+description: Rawler is a Ruby library that crawls your website and see the status code of each of your links. Useful for finding dead links.
+email: 
+- info@oscardelben.com
+executables: 
+- rawler
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- bin/rawler
+- lib/rawler/base.rb
+- lib/rawler/crawler.rb
+- lib/rawler.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/unit/base_spec.rb
+- spec/unit/crawler_spec.rb
+- tasks/rspec.rake
+has_rdoc: true
+homepage: http://github.com/#{github_username}/#{project_name}
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: oscardelben
+rubygems_version: 1.4.1
+signing_key: 
+specification_version: 3
+summary: Rawler is a Ruby library that crawls your website and see the status code of each of your links
+test_files: []
+