rawler 0.1.3 → 0.1.4

data/Gemfile

@@ -7,8 +7,9 @@ source "http://rubygems.org"
 # Include everything needed to run rake, tests, features, etc.
 gem 'nokogiri'
 
-
-group :development do
+group :development, :test do
+  gem 'fakeweb'
+  gem "rspec"
   gem "shoulda", ">= 0"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.4"

data/Gemfile.lock

@@ -1,14 +1,24 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    diff-lcs (1.1.3)
+    fakeweb (1.3.0)
     git (1.2.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
     nokogiri (1.5.0)
-    rake (0.9.2)
-    rcov (0.9.9)
+    rake (0.9.2.2)
+    rcov (0.9.11)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
     shoulda (2.11.3)
 
 PLATFORMS
@@ -16,7 +26,9 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  fakeweb
   jeweler (~> 1.6.4)
   nokogiri
   rcov
+  rspec
   shoulda

data/VERSION

@@ -1 +1 @@
-0.1.3
+0.1.4

data/lib/rawler.rb

@@ -1,82 +1,25 @@
-# **Rawler** is a command line tool for finding broken links on your website.
-# You can install Rawler by running:
-#
-#   gem install rawler
-#
-# To use Rawler type:
-#
-#   rawler example.com
-# 
-# Type `rawler -h` to see all the available options (including basic auth support).
-
-#### Prerequisites
-
-
 require 'rubygems'
-
-# We use [net/https](http://www.ruby-doc.org/stdlib/libdoc/net/http/rdoc/index.html) for making requests.
-
 require 'net/https'
-
-# We use [nokogiri](http://nokogiri.org/) for parsing web pages.
-
 require 'nokogiri'
-
-# We use the [logger](http://www.ruby-doc.org/stdlib/libdoc/logger/rdoc/) utility for handling the output.
-
 require 'logger'
-
-# We require [rawler/core_extensions](rawler/core_extensions.html) which includes some core extensions we need.
-
 require 'rawler/core_extensions'
-
-#### The Rawler module
-
-# The Rawler module itself is very simple, and it's only used for storing configuration data like the url that we want to fetch, basic username and password.
-
 module Rawler
   VERSION = "#{File.read(File.expand_path(File.dirname(__FILE__)) + '/../VERSION')}"
-  
-  # `output` is where we want to direct output. It's set to `$stdout` by default.
 
   mattr_accessor :output
-
-  # `url` is the url that we want to fetch. We need to keep track of it when parsing other pages to see if they are of the same domain.
-
   mattr_accessor :url
-
-  # The time we wait between requests, default 3. We don't want to send too many requests to your website!
-
   mattr_accessor :wait
-  
-  # Username and Password for basic auth, if needed.
-
   mattr_accessor :username, :password
-
-  # Log switch
-
   mattr_accessor :log
 
-  # Here we autoload when needed the specific namespaces.
-
-  # [Rawler::Base](rawler/base.html) is responsible for validating all the pages in a domain. It's where all the magic happens.
-
   autoload :Base, "rawler/base"
-
-  # [Rawler::Crawler](rawler/crawler.html) is responsible for parsing links inside a page.
-
   autoload :Crawler, "rawler/crawler"
-
-  # [Rawler::Request](rawler/reqeust.html) contains some helper methods for performing requests.
-
   autoload :Request, "rawler/request"
 
-  # We overwrite url= to automatically add `http://` if needed so that you can simply type `rawler example.com` in the command line.
-
   def self.url=(url)
     url.strip!
 
-    if (url =~ /http:\/\//) != 0
+    if (url =~ /http[s]?:\/\//) != 0
       url = 'http://' + url
     end
 

data/lib/rawler/base.rb

@@ -1,18 +1,7 @@
-#### Rawler workflow
-
-# `Rawler::Base` is where all the heavy work is being made.
-# When you call `rawler somesite.com`, we create an instance of Rawler::Base and then call `validate`, which recursively validates all the links relative to the domain that we specified.
-
 module Rawler
-  
   class Base
-    
-    # `responses` is used to keep track of which links we have already parsed, so that we wont parse them again and again.
-    # TODO: rename `responses` to something more meaningful.
 
     attr_accessor :responses
-    
-    # When we instantiate `Rawler::Base` we set some options according to what you specified on the command line.
 
     def initialize(url, output, options={})
       @responses = {}
@@ -26,18 +15,13 @@ module Rawler
       Rawler.log      = options[:log]
       @logfile = File.new("rawler_log.txt", "w") if Rawler.log
     end
-    
-    # The method used to start the real validation process
 
     def validate
       validate_links_in_page(Rawler.url)
       @logfile.close if Rawler.log
     end
-    
+
     private
-    
-    # We ask [Rawler::Crawler](crawler.html) for all the links in page and then validate each of them individually.
-    # We then sleep for the value of `Rawler.wait` (default 3) between each request to avoid dossing your server.
 
     def validate_links_in_page(page)
       Rawler::Crawler.new(page).links.each do |page_url|
@@ -45,8 +29,6 @@ module Rawler
         sleep(Rawler.wait)
       end
     end
-    
-    # If we haven't validated the page yet, we check its status code and then validate all the links in the page if it's in the same domain
 
     def validate_page(page_url, from_url)
       if not_yet_parsed?(page_url)
@@ -54,22 +36,12 @@ module Rawler
         validate_links_in_page(page_url) if same_domain?(page_url)
       end
     end
-    
-    # This is where we check the specific page status.
 
     def add_status_code(link, from_url)
       response = Rawler::Request.get(link)
 
-      # We follow a redirect if necessary.
-
       validate_page(response['Location'], from_url) if response['Location']
-      
-      # We inform the user about what we got.
-
       record_response(response.code, link, from_url, response['Location'])
-
-      # We add the current page to `responses` to avoid parsing it again/
-
       responses[link] = { :status => response.code.to_i }
     rescue Errno::ECONNREFUSED
       error("Connection refused - #{link} - Called from: #{from_url}")
@@ -79,41 +51,28 @@ module Rawler
     rescue Exception
       error("Unknown error - #{link} - Called from: #{from_url}")
     end
-    
-    # Some helper methods
 
     def same_domain?(link)
       URI.parse(Rawler.url).host == URI.parse(link).host
     end
-    
+
     def not_yet_parsed?(link)
       responses[link].nil?
     end
-    
+
     def error(message)
       Rawler.output.error(message)
     end
 
-    # We use this method to inform the user of a page status
-    
     def record_response(code, link, from_url, redirection=nil)
-
-      # By default, we just give the status code and the page url
-
       message = "#{code} - #{link}"
 
-      # If the status code is more or equal than 300, we also add which url linked the current page
-
       if code.to_i >= 300
         message += " - Called from: #{from_url}"
       end
 
-      # We add information about redirects, if a redirect was set
-          
       message += " - Following redirection to: #{redirection}" if redirection
 
-      # Depending on the status code, we use a different method of logger.
-
       code = code.to_i
       case code / 100
       when 1,2
@@ -127,7 +86,5 @@ module Rawler
       end
       @logfile.puts(message) if Rawler.log
     end
-    
   end
-  
 end

data/lib/rawler/core_extensions.rb

@@ -1,3 +1 @@
-# We load the [module](core_extensions/module.html) core extension
-
 require 'rawler/core_extensions/module'

data/lib/rawler/core_extensions/module.rb

@@ -1,5 +1,3 @@
-# Add `attr_accessor` like methods to modules
-
 class Module
   def mattr_reader(*syms)
     syms.each do |sym|
@@ -8,7 +6,7 @@ class Module
         unless defined? @@#{sym}
           @@#{sym} = nil
         end
-        
+
         def self.#{sym}
           @@#{sym}
         end
@@ -19,18 +17,18 @@ class Module
       EOS
     end
   end
-  
+
   def mattr_writer(*syms)
     syms.each do |sym|
       class_eval(<<-EOS, __FILE__, __LINE__)
         unless defined? @@#{sym}
           @@#{sym} = nil
         end
-        
+
         def self.#{sym}=(obj)
           @@#{sym} = obj
         end
-        
+
         #{"
         def #{sym}=(obj)
           @@#{sym} = obj
@@ -39,7 +37,7 @@ class Module
       EOS
     end
   end
-  
+
   def mattr_accessor(*syms)
     mattr_reader(*syms)
     mattr_writer(*syms)

data/lib/rawler/crawler.rb

@@ -1,40 +1,23 @@
-# `Rawler::Crawler` is responsible for parsing links inside a page
-
 module Rawler
-  
   class Crawler
-    
-    # An instance of Rawler::Crawler has a url which represents the url for which we want to parse links.
 
     attr_accessor :url
 
-    # We want to skip some kind of formats
-
-    SKIP_FORMATS = /^(javascript|mailto)/
- 
-    # To use this class, just pass it a url
+    SKIP_FORMATS = /^(javascript|mailto|callto)/
 
     def initialize(url)
       @url = url.strip
     end
 
-    # And then call `links` to get its links.
-    
     def links
-      # If the url is different than the main Rawler.url, or if the page is not html, we return an empty array
       if different_domain?(url, Rawler.url) || not_html?(url)
         return []
       end
-      
-      # Otherwise we fetch the page
 
       response = Rawler::Request.get(url)
 
-      # And kindly ask nokogiri to convert it for us
-      
       doc = Nokogiri::HTML(response.body)
 
-      # We then do some magic, search all the links in the document that contain a valid link, and return them.
       doc.css('a').map { |a| a['href'] }.select { |url| !url.nil? }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
     rescue Errno::ECONNREFUSED
       write("Couldn't connect to #{url}")
@@ -43,47 +26,39 @@ module Rawler
       write("Connection to #{url} timed out")
       []
     end
-    
+
     private
-    
-    # Here's how we transform a relative url to an absolute url
 
     def absolute_url(path)
-      # First, encode the url
       path = URI.encode(path.strip, Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}#]"))
 
-      # if the url contains a scheme that means it's already absolute
       if URI.parse(path).scheme
         path
       else
-        # Otherwise we merge `url` to get the absolute url
         URI.parse(url).merge(path).to_s
       end
     rescue URI::InvalidURIError
       write("Invalid url: #{path} - Called from: #{url}")
       nil
     end
-    
-    # Some helper methods
 
     def write(message)
       Rawler.output.error(message)
     end
-        
+
     def different_domain?(url_1, url_2)
       URI.parse(url_1).host != URI.parse(url_2).host
     end
-    
+
     def not_html?(url)
       Rawler::Request.head(url).content_type != 'text/html'
     end
-    
+
     def valid_url?(url)
       return false unless url
-      
       url.strip!
+
       scheme = URI.parse(url).scheme
-      
       if ['http', 'https'].include?(scheme)
         true
       else
@@ -95,7 +70,5 @@ module Rawler
       false
        write("Invalid url - #{url}")
     end
-      
   end
-  
 end

data/lib/rawler/request.rb

@@ -1,25 +1,20 @@
-# `Rawler::Request` contains some abstraction for making web requests, like automatically adding ssl and basic auth.
-
 module Rawler
-  
   class Request
-   
     class << self
-      
+
       def get(url)
         perform_request(:get, url)
       end
-      
+
       def head(url)
         perform_request(:head, url)
       end
-      
+
       private
-      
+
       def perform_request(method, url)
         uri = URI.parse(url)
 
-        # Use http_proxy if set
         proxy = URI.parse(ENV['http_proxy']) if ENV['http_proxy'] rescue nil
         if proxy
           http = Net::HTTP::Proxy(proxy.host, proxy.port).new(uri.host, uri.port)
@@ -28,16 +23,13 @@ module Rawler
         end
         http.use_ssl = (uri.scheme == 'https')
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        
+
         path = (uri.path.size == 0)  ? "/" : uri.path
-        
+
         request = Net::HTTP::Get.new(path)
         request.basic_auth(Rawler.username, Rawler.password)
         http.request(request)
       end
-      
     end
-    
   end
-  
 end

data/rawler.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "rawler"
-  s.version = "0.1.3"
+  s.version = "0.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Oscar Del Ben"]
-  s.date = "2011-11-11"
+  s.date = "2012-02-27"
   s.description = "Rawler is a tool that crawls the links of your website"
   s.email = "info@oscardelben.com"
   s.executables = ["rawler"]
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/oscardelben/rawler"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.11"
+  s.rubygems_version = "1.8.10"
   s.summary = "Rawler is a tool that crawls the links of your website"
 
   if s.respond_to? :specification_version then
@@ -53,12 +53,16 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
@@ -66,6 +70,8 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])

data/spec/lib/rawler/crawler_spec.rb

@@ -159,6 +159,25 @@ describe Rawler::Crawler do
         crawler.links
       end
     end
+    
+    context "callto" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"callto:home22\">foo</a><a name=\"foo\">" }
+    
+      before(:each) do
+        register(url, content)
+      end
+    
+      it "should return empty links" do
+        crawler.links.should == []
+      end
+
+      it "should not report the error" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+    end
 
   end
 

data/spec/lib/rawler_spec.rb

@@ -20,6 +20,27 @@ describe Rawler::Base do
       Rawler::Base.new(original, output)
       Rawler.url.should == expected
     end
+    
+    it "should auto prepend http" do
+      original = 'example.com'
+      expected = 'http://example.com'
+      Rawler::Base.new(original, output)
+      Rawler.url.should == expected
+    end
+    
+    it "should not auto prepend http when already http" do
+      original = 'http://example.com'
+      expected = 'http://example.com'
+      Rawler::Base.new(original, output)
+      Rawler.url.should == expected
+    end
+    
+    it "should not auto prepend http when https" do
+      original = 'https://example.com'
+      expected = 'https://example.com'
+      Rawler::Base.new(original, output)
+      Rawler.url.should == expected
+    end
   end
   
   describe "validate_links" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rawler
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 1
-  - 3
-  version: 0.1.3
+  - 4
+  version: 0.1.4
 platform: ruby
 authors: 
 - Oscar Del Ben
@@ -15,10 +15,10 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-11 00:00:00 Z
+date: 2012-02-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -27,12 +27,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  version_requirements: *id001
+  requirement: *id001
+  type: :runtime
+  prerelease: false
   name: nokogiri
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id002
+  type: :development
   prerelease: false
-  type: :runtime
+  name: fakeweb
 - !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -41,12 +55,26 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  version_requirements: *id002
-  name: shoulda
+  requirement: *id003
+  type: :development
   prerelease: false
+  name: rspec
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
   type: :development
+  prerelease: false
+  name: shoulda
 - !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -57,12 +85,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  version_requirements: *id003
-  name: bundler
-  prerelease: false
+  requirement: *id005
   type: :development
+  prerelease: false
+  name: bundler
 - !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -73,12 +101,12 @@ dependencies:
         - 6
         - 4
         version: 1.6.4
-  version_requirements: *id004
-  name: jeweler
-  prerelease: false
+  requirement: *id006
   type: :development
+  prerelease: false
+  name: jeweler
 - !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -87,10 +115,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  version_requirements: *id005
-  name: rcov
-  prerelease: false
+  requirement: *id007
   type: :development
+  prerelease: false
+  name: rcov
 description: Rawler is a tool that crawls the links of your website
 email: info@oscardelben.com
 executables: 
@@ -153,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Rawler is a tool that crawls the links of your website