link-checker 0.2.0 → 0.3.0

data/Gemfile

@@ -2,6 +2,7 @@ source :rubygems
 
 gem "nokogiri", "~> 1.5.5"
 gem "colorize", "~> 0.5.8"
+gem "anemone", "~> 0.7.2"
 
 group(:test, :development) do
   gem "rspec", "~> 2.11.0"

data/Gemfile.lock

@@ -1,6 +1,9 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    anemone (0.7.2)
+      nokogiri (>= 1.3.0)
+      robotex (>= 1.0.0)
     colorize (0.5.8)
     columnize (0.3.6)
     debugger (1.2.0)
@@ -25,6 +28,7 @@ GEM
     rdoc (3.12)
       json (~> 1.4)
     redcarpet (2.1.1)
+    robotex (1.0.0)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
       rspec-expectations (~> 2.11.0)
@@ -43,6 +47,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  anemone (~> 0.7.2)
   colorize (~> 0.5.8)
   debugger
   fakeweb (~> 1.3.0)

data/README.md

@@ -14,7 +14,7 @@ Then ```bundle install``` to install the gem.
 
 ## Usage
 
-You can use the ```check-links``` command to specify any directory to scan for HTML files.
+You can use the ```check-links [PATH]``` command to specify any directory to scan for HTML files.  The default path is ```./```.
 
 For example, to check the links for an Octopress site:
 
@@ -30,4 +30,8 @@ The ```link-checker``` gem uses [RSpec](http://rspec.info) for testing and has 1
 
 Run the specs with:
 
-    rake spec
+    rake spec
+
+## API Documentation
+
+The Yardoc documenation is hosted on [RubyDoc.info](http://rubydoc.info/github/endymion/link-checker/frames).

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/bin/check-links

@@ -3,6 +3,6 @@
 require 'link_checker'
 
 # Assume the 'public' directory is the site unless otherwise specified.
-site_path = ARGV[0] || './'
+target = ARGV[0] || './'
 
-LinkChecker.new(site_path).check_links
+LinkChecker.new(target).check_uris

data/lib/link_checker.rb

@@ -4,29 +4,28 @@ require 'net/http'
 require 'net/https'
 require 'uri'
 require 'colorize'
+require 'anemone'
 
 class LinkChecker
 
-  def initialize(target_path)
-    @target_path = target_path
+  def initialize(target)
+    @target = target
   end
 
-  def find_html_files
-    Find.find(@target_path).map {|path|
+  def html_file_paths
+    Find.find(@target).map {|path|
       FileTest.file?(path) && (path =~ /\.html?$/) ? path : nil
     }.reject{|path| path.nil?}
   end
 
-  def self.find_external_links(file_path)
-    Nokogiri::HTML(open(file_path)).css('a').
-      select do |link|
+  def self.external_link_uri_strings(source)
+    Nokogiri::HTML(source).css('a').select {|link|
         !link.attribute('href').nil? &&
         link.attribute('href').value =~ /^https?\:\/\//
-      end
+    }.map{|link| link.attributes['href'].value}
   end
 
-  def self.check_link(uri, redirected=false)
-    uri = URI.parse(uri)
+  def self.check_uri(uri, redirected=false)
     http = Net::HTTP.new(uri.host, uri.port)
     http.use_ssl = true if uri.scheme == "https"
     http.start do
@@ -35,68 +34,110 @@ class LinkChecker
         case response
         when Net::HTTPSuccess then
           if redirected
-            return Redirect.new(uri)
+            return Redirect.new(:final_destination_uri_string => uri.to_s)
           else
-            return Good.new
+            return Good.new(:uri_string => uri.to_s)
           end
         when Net::HTTPRedirection then
-          return self.check_link(response['location'], true)
+          return self.check_uri(URI(response['location']), true)
         else
-          raise Error.new(response)
+          return Error.new(:uri_string => uri.to_s, :response => response)
         end
       end
     end
   end
 
-  def check_links
-    find_html_files.each do |file|
-      bad_checks = []
-      warnings = []
-      self.class.find_external_links(file).each do |link|
-        uri = link.attribute('href').value
+  def check_uris
+    if @target =~ /^https?\:\/\//
+      check_uris_by_crawling
+    else
+      check_uris_in_files
+    end
+  end
+
+  def check_uris_by_crawling
+    threads = []
+    Anemone.crawl(@target) do |anemone|
+      anemone.storage = Anemone::Storage.PStore('link-checker-crawled-pages.pstore')
+      anemone.on_every_page do |crawled_page|
+        threads << start_link_check_thread(crawled_page.body, crawled_page.url.to_s)
+      end
+    end
+    threads.each{|thread| thread.join }
+  end
+
+  def check_uris_in_files
+    threads = []
+    html_file_paths.each do |file|
+      threads << start_link_check_thread(open(file), file)
+    end
+    threads.each{|thread| thread.join }
+  end
+
+  def start_link_check_thread(source, source_name)
+    Thread.new do
+      results = self.class.external_link_uri_strings(source).map do |uri_string|
         begin
-          response = self.class.check_link(uri)
-          if response.class.eql? Redirect
-            warnings << { :link => link, :response => response }
-          end
+          uri = URI(uri_string)
+          response = self.class.check_uri(uri)
+          { :uri_string => uri_string, :response => response }
         rescue => error
-          bad_checks << { :link => link, :response => error }
+          { :uri_string => uri_string, :response => Error.new(:error => error.to_s) }
         end
       end
+      report_results(source_name, results)
+    end
+  end
       
+  def report_results(file, results)
+    bad_checks = results.select{|result| result[:response].class.eql? Error}
+    warnings = results.select{|result| result[:response].class.eql? Redirect}
+    Thread.exclusive do
       if bad_checks.empty?
+        message = "Checked: #{file}"
         if warnings.empty?
-          puts "Checked: #{file}".green
+          puts message.green
         else
-          puts "Checked: #{file}".yellow
+          puts message.yellow
         end
         warnings.each do |warning|
-          puts "   Warning: #{warning[:link].attribute('href').value}".yellow
-          puts "     Redirected to: #{warning[:response].final_destination.to_s}".yellow
+          puts "   Warning: #{warning[:uri_string]}".yellow
+          puts "     Redirected to: #{warning[:response].final_destination_uri_string}".yellow
         end
       else
         puts "Problem: #{file}".red
         bad_checks.each do |check|
-          puts "   Link: #{check[:link].attribute('href').value}".red
-          puts "     Response: #{check[:response].response.inspect}".red
+          puts "   Link: #{check[:uri_string]}".red
+          puts "     Response: #{check[:response].error.to_s}".red
         end
       end
     end
   end
 
-  class Good; end
+  class Result
+    attr_reader :uri_string
+    def initialize(params)
+      @uri_string = params[:uri_string]
+    end
+  end
+
+  class Good < Result
+  end
 
-  class Redirect
-    attr_reader :final_destination
-    def initialize(final_destination)
-      @final_destination = final_destination
+  class Redirect < Result
+    attr_reader :good
+    attr_reader :final_destination_uri_string
+    def initialize(params)
+      @final_destination_uri_string = params[:final_destination_uri_string]
+      @good = params[:good]
+      super(params)
     end
   end
 
-  class Error < StandardError
-    attr_accessor :response
-    def initialize(response)
-      @response = response
+  class Error < Result
+    attr_reader :error
+    def initialize(params)
+      @error = params[:error]
     end
   end
 

data/link-checker.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "link-checker"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ryan Alyn Porter"]
-  s.date = "2012-10-04"
+  s.date = "2012-10-05"
   s.description = "Check the links in a web site before deploying, using Nokogiri."
   s.executables = ["check-links"]
   s.extra_rdoc_files = [
@@ -368,7 +368,7 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/endymion/link-checker"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.24"
   s.summary = "Check the links in a web site before deploying."
 
   if s.respond_to? :specification_version then
@@ -377,6 +377,7 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, ["~> 1.5.5"])
       s.add_runtime_dependency(%q<colorize>, ["~> 0.5.8"])
+      s.add_runtime_dependency(%q<anemone>, ["~> 0.7.2"])
       s.add_development_dependency(%q<rspec>, ["~> 2.11.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
       s.add_development_dependency(%q<simplecov>, ["~> 0.6.4"])
@@ -387,6 +388,7 @@ Gem::Specification.new do |s|
     else
       s.add_dependency(%q<nokogiri>, ["~> 1.5.5"])
       s.add_dependency(%q<colorize>, ["~> 0.5.8"])
+      s.add_dependency(%q<anemone>, ["~> 0.7.2"])
       s.add_dependency(%q<rspec>, ["~> 2.11.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
       s.add_dependency(%q<simplecov>, ["~> 0.6.4"])
@@ -398,6 +400,7 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<nokogiri>, ["~> 1.5.5"])
     s.add_dependency(%q<colorize>, ["~> 0.5.8"])
+    s.add_dependency(%q<anemone>, ["~> 0.7.2"])
     s.add_dependency(%q<rspec>, ["~> 2.11.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
     s.add_dependency(%q<simplecov>, ["~> 0.6.4"])

data/spec/link-checker_spec.rb

@@ -3,88 +3,137 @@ require 'link_checker'
 
 describe LinkChecker do
 
-  before(:all) do
-    @site_path = 'spec/test-site/public/'
-  end
+  describe "scans a file path and" do
 
-  it "finds all of the HTML files in the target path." do
-    files = LinkChecker.new(@site_path).find_html_files
-    files.size.should == 3
-  end
+    before(:all) do
+      @site_path = 'spec/test-site/public/'
+    end
+
+    it "finds all of the HTML files in the target path." do
+      files = LinkChecker.new(@site_path).html_file_paths
+      files.size.should == 3
+    end
+
+    it "finds all of the external links in an HTML file." do
+      links = LinkChecker.external_link_uri_strings(
+        open('spec/test-site/public/blog/2012/10/02/a-list-of-links/index.html'))
+      links.size.should == 4
+    end
+
+    it "finds all of the external links in a string." do
+      links = LinkChecker.external_link_uri_strings(
+        open('spec/test-site/public/blog/2012/10/02/a-list-of-links/index.html').read)
+      links.size.should == 4
+    end
 
-  it "finds all of the external links in an HTML file." do
-    links = LinkChecker.find_external_links(
-      'spec/test-site/public/blog/2012/10/02/a-list-of-links/index.html')
-    links.size.should == 4
   end
 
   describe "checks links and" do
 
     before(:all) do
-      @good_uri = 'http://goodlink.com'
-      FakeWeb.register_uri(:any, @good_uri, :body => "Yay it worked.")
+      @good_uri = URI('http://goodlink.com')
+      FakeWeb.register_uri(:any, @good_uri.to_s, :body => "Yay it worked.")
 
-      @bad_uri = 'http://brokenlink.com'
-      FakeWeb.register_uri(:get, @bad_uri,
+      @bad_uri = URI('http://brokenlink.com')
+      FakeWeb.register_uri(:get, @bad_uri.to_s,
         :body => "File not found", :status => ["404", "Missing"])
 
-      @redirect_uri = 'http://redirect.com'
+      @redirect_uri = URI('http://redirect.com')
     end
 
     it "declares good links to be good." do
-      LinkChecker.check_link(@good_uri).class.should be LinkChecker::Good
+      LinkChecker.check_uri(@good_uri).class.should be LinkChecker::Good
     end
 
     it "declares bad links to be bad." do
-      expect { LinkChecker.check_link(@bad_uri) }.to(
-        raise_error(LinkChecker::Error))
+      LinkChecker.check_uri(@bad_uri).class.should be LinkChecker::Error
     end
 
     describe "follows redirects to the destination and" do
 
       it "declares good redirect targets to be good." do
-        FakeWeb.register_uri(:get, @redirect_uri,
-          :location => @good_uri, :status => ["302", "Moved"])
-        result = LinkChecker.check_link(@redirect_uri)
+        FakeWeb.register_uri(:get, @redirect_uri.to_s,
+          :location => @good_uri.to_s, :status => ["302", "Moved"])
+        result = LinkChecker.check_uri(@redirect_uri)
         result.class.should be LinkChecker::Redirect
-        result.final_destination.to_s.should == @good_uri
+        result.final_destination_uri_string.should == @good_uri.to_s
       end
 
       it "declares bad redirect targets to be bad." do
-        FakeWeb.register_uri(:get, @redirect_uri,
-          :location => @bad_uri, :status => ["302", "Moved"])
-        expect { LinkChecker.check_link(@redirect_uri) }.to(
-          raise_error(LinkChecker::Error))
+        FakeWeb.register_uri(:get, @redirect_uri.to_s,
+          :location => @bad_uri.to_s, :status => ["302", "Moved"])
+        result = LinkChecker.check_uri(@redirect_uri)
+        result.class.should be LinkChecker::Error
       end
 
     end
 
   end
 
-  describe "prints output" do
+  describe "scans a file path and prints output" do
 
-    it "prints green when the links are all good." do
-      LinkChecker.stub(:check_link) { LinkChecker::Good.new }
-      $stdout.should_receive(:puts).with(/Checked/i).exactly(3).times
-      LinkChecker.new(@site_path).check_links
+    before(:each) do
+      LinkChecker.any_instance.stub(:html_file_paths) {
+        ['spec/test-site/public/blog/2012/10/02/a-list-of-links/index.html'] }
+      LinkChecker.stub(:external_link_uri_strings).and_return(['http://something.com'])
     end
 
-    it "prints red when the links are all bad." do
-      LinkChecker.stub(:check_link).and_raise LinkChecker::Error.new('blah')
-      $stdout.should_receive(:puts).with(/Problem/i).exactly(3).times
-      $stdout.should_receive(:puts).with(/Link/i).at_least(3).times
-      $stdout.should_receive(:puts).with(/Response/i).at_least(3).times
-      LinkChecker.new(@site_path).check_links
+    it "prints green when the links are good." do
+      LinkChecker.stub(:check_uri) do
+        LinkChecker::Good.new(:uri_string => 'http://something.com')
+      end
+      $stdout.should_receive(:puts).with(/Checked/i).once
+      LinkChecker.new(@site_path).check_uris
+    end
+
+    it "prints red when the links are bad." do
+      LinkChecker.stub(:check_uri) do
+        LinkChecker::Error.new(
+          :uri_string => 'http://something.com',
+          :response => 'No.'
+        )
+      end
+      $stdout.should_receive(:puts).with(/Problem/i).once
+      $stdout.should_receive(:puts).with(/Link/i).once
+      $stdout.should_receive(:puts).with(/Response/i).once
+      LinkChecker.new(@site_path).check_uris
     end
 
     it "prints yellow warnings when the links redirect." do
-      LinkChecker.stub(:check_link) { LinkChecker::Redirect.new('http://somewhere') }
-      $stdout.should_receive(:puts).with(/Checked/i).exactly(3).times
-      $stdout.should_receive(:puts).with(/Warning/i).at_least(3).times
-      $stdout.should_receive(:puts).with(/Redirected/i).at_least(3).times
-      LinkChecker.new(@site_path).check_links
+      LinkChecker.stub(:check_uri) do
+        LinkChecker::Redirect.new(
+          :uri_string => 'http://something.com',
+          :final_desination => 'http://something-else.com'
+        )
+      end
+      $stdout.should_receive(:puts).with(/Checked/i).once
+      $stdout.should_receive(:puts).with(/Warning/i).once
+      $stdout.should_receive(:puts).with(/Redirected/i).once
+      LinkChecker.new(@site_path).check_uris
     end
 
   end
 
+  describe "prints output for invalid links and" do
+
+    it "declares them to be bad." do
+      LinkChecker.stub(:external_link_uri_strings).and_return(['hQQp://!!!.com'])
+      $stdout.should_receive(:puts).with(/Problem/i).once
+      $stdout.should_receive(:puts).with(/Link/i).once
+      $stdout.should_receive(:puts).with(/Response/i).once
+      thread = LinkChecker.new(@site_path).start_link_check_thread('<html></html>', 'source.html')
+      thread.join
+    end
+
+  end
+
+  it "crawls a web site." do
+    LinkChecker.stub(:external_link_uri_strings).and_return(['http://something.com'])
+    LinkChecker.stub(:check_uri) do
+      LinkChecker::Good.new(:uri_string => 'http://something.com')
+    end
+    $stdout.should_receive(:puts).with(/Checked/i).once
+    LinkChecker.new('http://pretty-sure-this-is-not-real.com').check_uris
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: link-checker
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-04 00:00:00.000000000 Z
+date: 2012-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70123915371780 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,21 +21,47 @@ dependencies:
         version: 1.5.5
   type: :runtime
   prerelease: false
-  version_requirements: *70123915371780
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
 - !ruby/object:Gem::Dependency
   name: colorize
-  requirement: &70123915371000 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.8
+- !ruby/object:Gem::Dependency
+  name: anemone
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.2
   type: :runtime
   prerelease: false
-  version_requirements: *70123915371000
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.2
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70123915370260 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +69,15 @@ dependencies:
         version: 2.11.0
   type: :development
   prerelease: false
-  version_requirements: *70123915370260
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.0
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70123915386020 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +85,15 @@ dependencies:
         version: 1.8.4
   type: :development
   prerelease: false
-  version_requirements: *70123915386020
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.4
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70123915385300 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +101,15 @@ dependencies:
         version: 0.6.4
   type: :development
   prerelease: false
-  version_requirements: *70123915385300
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.4
 - !ruby/object:Gem::Dependency
   name: fakeweb
-  requirement: &70123915384680 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +117,15 @@ dependencies:
         version: 1.3.0
   type: :development
   prerelease: false
-  version_requirements: *70123915384680
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &70123915384020 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +133,15 @@ dependencies:
         version: 0.8.2.1
   type: :development
   prerelease: false
-  version_requirements: *70123915384020
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.2.1
 - !ruby/object:Gem::Dependency
   name: redcarpet
-  requirement: &70123915383280 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +149,15 @@ dependencies:
         version: 2.1.1
   type: :development
   prerelease: false
-  version_requirements: *70123915383280
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
 - !ruby/object:Gem::Dependency
   name: debugger
-  requirement: &70123915382660 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +165,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70123915382660
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Check the links in a web site before deploying, using Nokogiri.
 email: 
 executables:
@@ -481,7 +542,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4067428474501451879
+      hash: 2583548711827619291
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -490,7 +551,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Check the links in a web site before deploying.