site_checker 0.1.1 → 0.2.0.pre

data/lib/site_checker.rb

@@ -1,224 +1,108 @@
-require 'nokogiri'
 require 'open-uri'
+require 'nokogiri'
 
-class SiteChecker
-  attr_accessor :problems
-  attr_accessor :ignore_list, :visit_references, :max_recursion_depth
-
-  def initialize()
-    yield self if block_given?  
-    @ignore_list ||= []
-    @visit_references ||= false
-    @max_recursion_depth ||= -1
-  end
-
-  def check(url, root)
-    @visits = {}
-    @problems = {}
-    @recursion_depth = 0
-    
-    @root = root
-
-    register_visit(:local_page, url)
-    process_local_page(url, nil)
-  end
-
-  def local_pages
-    @visits[:local_page]
-  end
-
-  def remote_pages
-    @visits[:remote_page]
-  end
-
-  def local_images
-    @visits[:local_image]
-  end
-
-  def remote_images
-    @visits[:remote_image]
-  end
-
-  private
-  def process_local_page(url, parent_url)
-    links = collect_links(url, parent_url)
-
-    filter_out_working_anchors!(links)
-    report_and_remove_anchors!(links, parent_url)
-
-    links.each do |link, kind|
-      if kind != :anchor
-        visit(kind, url, link) unless visited?(kind, link)
-      else
-      end
+require 'site_checker/io/content_from_file_system'
+require 'site_checker/io/content_from_web'
+require 'site_checker/parse/page'
+require 'site_checker/link'
+require 'site_checker/link_collector'
+require 'site_checker/dsl'
+
+module SiteChecker
+  class << self
+    attr_accessor :ignore_list
+    attr_accessor :visit_references
+    attr_accessor :max_recursion_depth
+    attr_accessor :dsl_enabled
+    attr_reader   :link_collector
+
+    ##
+    # The following configuration options, which can be used together, are available:
+    #
+    # - ignoring certain links:
+    #
+    #     SiteChecker.configure do |config|
+    #       config.ignore_list = ["/", "/atom.xml"]
+    #     end
+    #
+    # - visit the external references as well:
+    #
+    #     SiteChecker.configure do |config|
+    #       config.visit_references = true
+    #     end
+    #
+    # - set the depth of the recursion:
+    #
+    #     SiteChecker.configure do |config|
+    #       config.max_recursion_depth = 3
+    #     end
+    def configure
+      yield self
     end
-  end
-
-  def register_visit(kind, link)
-    @visits[kind] = [] unless @visits.has_key?(kind)
-    @visits[kind] << link
-  end
 
-  def visited?(kind, link)
-    @visits[kind] = [] unless @visits.has_key?(kind)
-    @visits[kind].include?(link)
-  end
-
-  def visit(kind, parent_url, link)
-      register_visit(kind, link)
-      if kind != :local_page
-        open_reference(kind, link, parent_url)
-      else
-        unless stop_recursion?
-          @recursion_depth += 1
-          process_local_page(link, parent_url)
-          @recursion_depth -= 1
-        end
-      end
-  end
-
-  def open_reference(kind, link, parent_url)
-    content = nil
-    begin
-      if kind == :local_page
-        if URI(@root).absolute?
-          content = open(link)
-        else
-          link = add_index_html(link)
-          content = File.open(link).read
-        end
-      elsif kind == :local_image
-        if URI(@root).absolute?
-          open(link)
-        else
-          File.open(link)
-        end
-      elsif @visit_references
-        open(link)
-      end
-    rescue OpenURI::HTTPError => e
-      new_problem(strip_root(parent_url), "#{strip_root(link)} (#{e.message.strip})")
-    rescue Errno::ENOENT => e
-      link = remove_index_html(link) if kind == :local_page
-      new_problem(strip_root(parent_url), "#{strip_root(link)} (404 Not Found)")
-    rescue => e
-      new_problem(strip_root(parent_url), "#{strip_root(link)} (#{e.message.strip})")
+    ##
+    # Recursively visits the provided url looking for reference problems.
+    #
+    # @param [String] url where the processing starts
+    # @param [String] root the root URL of the site
+    #
+    def check(url, root)
+      create_instance
+      @link_collector.check(url, root)
     end
-    content
-  end
 
-  def filter_out_working_anchors!(links)
-    links.delete_if{ |link, kind| (kind == :local_page && has_anchor?(links, link)) }
-  end
-
-  def report_and_remove_anchors!(links, parent_url)
-    anchors = links.select {|link, kind| link.match(/^.+#.+$/) && kind == :local_page}
-    anchors.each do |anchor, kind|
-      new_problem(strip_root(parent_url), "#{strip_root(anchor)} (404 Not Found)")
-      links.delete(anchor)
+    ##
+    # Returns the Array of the visited local pages.
+    #
+    # @return [Array] list of the visited local pages
+    #
+    def local_pages
+      @link_collector.local_pages
     end
-  end
-
-  def has_anchor?(links, link)
-    anchor = link.gsub(/^.+#/, "")
-    links.has_key?(anchor) && links[anchor] == :anchor
-  end
-
-
-  def absolute_reference?(link)
-    link.start_with?(@root)
-  end
 
-  def relative_reference?(link)
-    link =~ /^\/.+/
-  end
-
-  def collect_links(url, parent_url)
-    links = {}
-    content = open_reference(:local_page, url, parent_url)
-    if content
-      doc = Nokogiri(content)
-      doc.xpath("//img").reject {|img| ignored?(img['src'])}.each do |img|
-        link_kind = detect_link_and_kind(img['src'], url, :remote_image, :local_image)
-        links.merge!(link_kind) unless link_kind.empty?    
-      end
-      doc.xpath("//a").reject {|a| ignored?(a['href'])}.each do |a|
-        link_kind = detect_link_and_kind(a['href'], url, :remote_page, :local_page)
-        links.merge!(link_kind) unless link_kind.empty?
-      end
-    
-      doc.xpath("//a").reject {|a| !a['id']}.each do |a|
-        links.merge!({a['id'] => :anchor})
-      end
+    ##
+    # Returns the Array of the visited remote (external) pages.
+    #
+    # @return [Array] list of the visited remote pages
+    #
+    def remote_pages
+      @link_collector.remote_pages
     end
-    links
-  end
 
-  def detect_link_and_kind(reference, url, external_kind, local_kind)
-    link_kind = {}
-    link = URI(strip_trailing_slash(reference))
-    if link.to_s.start_with?(@root)
-      new_problem(url, "#{link} (absolute path)")
-    else
-      if URI(reference).absolute?
-        link_kind[link.to_s] = external_kind
-      else
-        link_kind[create_absolute_reference(link.to_s)] = local_kind
-      end
+    ##
+    # Returns the Array of the visited local images.
+    #
+    # @return [Array] list of the visited local images
+    #
+    def local_images
+      @link_collector.local_images
     end
-    link_kind
-  end
-
-  def strip_trailing_slash(link)
-    link.gsub(/\/$/, "")
-  end
 
-  def strip_root(link)
-    if link
-      link.gsub(/^#{@root}[\/]?/, "")
-    else
-      ""
+    ##
+    # Returns the Array of the visited remote (external) images.
+    #
+    # @return [Array] list of the visited remote images
+    #
+    def remote_images
+      @link_collector.remote_images
     end
-  end
-
-  def add_index_html(path)
-    path.end_with?(".html") ? path : File.join(path, "index.html") 
-  end
-
-  def remove_index_html(path)
-    path.gsub(/\/index.html$/, "")
-  end
 
-  def create_absolute_reference(link)
-    root = URI(@root)
-    if root.absolute?
-      root.merge(link).to_s.gsub(/\/$/, "")
-    else
-      File.join(root.path, link)
+    ##
+    # Returns the Hash (:parent_url => [Array of problematic links]) of the problems.
+    #
+    # @return [Hash] the result of the check
+    #
+    def problems
+      @link_collector.problems
     end
-  end
-
-  def new_problem(url, message)
-    url = @root if url.empty?
-    @problems[url] = [] unless problems.has_key?(url)
-    @problems[url] << message
-  end
 
-  def ignored?(link)
-    if link
-      @ignore_list.include? link
-    else
-      true
-    end
-  end
-  
-  def stop_recursion?
-    if @max_recursion_depth == -1
-      false
-    elsif @max_recursion_depth > @recursion_depth
-      false
-    else
-      true
+    private
+    def create_instance
+      @link_collector = SiteChecker::LinkCollector.new do |config|
+        config.visit_references = @visit_references if @visit_references
+        config.ignore_list = @ignore_list if @ignore_list
+        config.max_recursion_depth = @max_recursion_depth if @max_recursion_depth
+      end
     end
   end
-end
+end

data/site_checker.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name        = 'site_checker'
+  s.version     = '0.2.0.pre'
+  s.date        = '2012-12-22'
+  s.summary     = "site_checker-#{s.version}"
+  s.description = "A simple tool for checking references on your website"
+  s.authors     = ["Zsolt Fabok"]
+  s.email       = 'me@zsoltfabok.com'
+  s.homepage    = 'https://github.com/ZsoltFabok/site_checker'
+  s.license     = 'BSD'
+
+  s.files         = `git ls-files`.split("\n").reject {|path| path =~ /\.gitignore$/ || path =~ /file$/ }
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency('rspec'  , '2.12.0')
+  s.add_development_dependency('webmock', '1.9.0')
+  s.add_development_dependency('rake'   , '10.0.3')
+  s.add_development_dependency('yard'   , '0.8.3')
+
+  s.add_runtime_dependency('nokogiri', '1.5.6')
+end

data/spec/dsl_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require 'site_checker/io/io_spec_helper'
+
+describe "DSL" do
+  include IoSpecHelper
+
+  before(:each) do
+    @test_url = "http://localhost:4000"
+    @root = "http://localhost:4000"
+  end
+
+  it "should forward all the method calls if DSL is enabled" do
+     SiteChecker.configure do |config|
+      config.dsl_enabled = true
+    end
+
+    local_pages   = mock()
+    local_images  = mock()
+    remote_pages  = mock()
+    remote_images = mock()
+    problems      = mock()
+
+    SiteChecker.should_receive(:check).with(@test_url, @root)
+    SiteChecker.should_receive(:local_pages).and_return(local_pages)
+    SiteChecker.should_receive(:remote_pages).and_return(remote_pages)
+    SiteChecker.should_receive(:local_images).and_return(local_images)
+    SiteChecker.should_receive(:remote_images).and_return(remote_images)
+    SiteChecker.should_receive(:problems).and_return(problems)
+
+    check_site(@test_url, @root)
+    collected_local_pages.should eql(local_pages)
+    collected_remote_pages.should eql(remote_pages)
+    collected_local_images.should eql(local_images)
+    collected_remote_images.should eql(remote_images)
+    collected_problems.should eql(problems)
+  end
+end

data/spec/integration_spec.rb

@@ -0,0 +1,191 @@
+require 'spec_helper'
+require 'site_checker/io/io_spec_helper'
+
+describe "Integration" do
+  include IoSpecHelper
+
+  before(:each) do
+    SiteChecker.configure do |config|
+      config.visit_references = true
+    end
+  end
+
+  describe "server based checking" do
+    before(:each) do
+      @test_url = "http://localhost:4000"
+      @root = "http://localhost:4000"
+    end
+
+    it "should visit the page" do
+      content = "<html></html>"
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.local_pages.should eql([@test_url])
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should check the link to an external page" do
+      content = "<html>text<a href=\"http://external.org/\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://external.org", 200, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.remote_pages.should eql(["http://external.org/" ])
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should not check the link to an external page if the reference checking is turned off" do
+      SiteChecker.configure do |config|
+        config.visit_references = false
+      end
+      content = "<html>text<a href=\"http://external.org/\"/></html>"
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem if the external link is dead" do
+      content = "<html>text<a href=\"http://external.org/\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://external.org", 404, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["http://external.org/ (404)"]})
+    end
+
+    it "should check the link to an external image" do
+      content = "<html>text<img src=\"http://external.org/a.png\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://external.org/a.png", 200, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should check the link to a local image" do
+      content = "<html>text<img src=\"/a.png\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("#{@test_url}/a.png", 200, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem if the image cannot be found" do
+      content = "<html>text<img src=\"http://external.org/a.png\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://external.org/a.png", 404, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["http://external.org/a.png (404)"]})
+    end
+
+    it "should report a problem for a local page with absolute path" do
+      content = "<html>text<a href=\"#{@test_url}/another\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("#{@test_url}/another", 200, "")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["#{@test_url}/another (absolute path)"]})
+    end
+
+    it "should report a problem for a local image with absolute path" do
+      content = "<html>text<img src=\"#{@test_url}/a.png\"/></html>"
+      webmock("#{@test_url}/a.png", 200, "")
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["#{@test_url}/a.png (absolute path)"]})
+    end
+
+    it "should filter out certain links" do
+      SiteChecker.configure do |config|
+        config.ignore_list = ["/atom.xml", "/"]
+      end
+      content = "<html>text<a href=\"/atom.xml\"/><br/><a href=\"/\"/></html>"
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should not report a valid internal anchor" do
+      content = "<html><a href=\"#goto\">goto</a>text<a id=\"goto\"></a></html>"
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report an invalid internal anchor" do
+      content = "<html><a href=\"#goto\">goto</a>text<a id=\"got\"></a></html>"
+      webmock(@test_url, 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["#goto (404 Not Found)"]})
+    end
+
+    it "should follow an external anchor to the external page" do
+      content = "<html><a href=\"http://example.org#goto\">goto</a></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://example.org", 200, content)
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should go down one level down for an internal page" do
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("#{@root}/one-level-down", 200, "<html></html>")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem with a linked local page" do
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("#{@root}/one-level-down", 404, "<html></html>")
+      SiteChecker.check(@test_url, @root)
+      SiteChecker.problems.should eql({@test_url => ["/one-level-down (404)"]})
+    end
+  end
+
+  describe "file system based checking" do
+    before(:each) do
+      @root = fs_test_path
+      clean_fs_test_path
+    end
+
+    it "should find a referenced page" do
+      @root = fs_test_path
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      filesystemmock("index.html", content)
+      filesystemmock("/one-level-down/index.html", content)
+      SiteChecker.check(fs_test_path, @root)
+      SiteChecker.local_pages.should eql([fs_test_path, "/one-level-down"])
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem when the local page cannot be found" do
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      filesystemmock("index.html", content)
+      SiteChecker.check(fs_test_path, @root)
+      SiteChecker.problems.should eql({fs_test_path => ["/one-level-down (404 Not Found)"]})
+    end
+
+    it "should use the local images" do
+      content = "<html>text<img src=\"/a.png\"/></html>"
+      filesystemmock("index.html", content)
+      filesystemmock("a.png", "")
+      SiteChecker.check(fs_test_path, @root)
+      SiteChecker.local_images.should eql(["/a.png"])
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem when the local image cannot be found" do
+      content = "<html>text<img src=\"/a.png\"/></html>"
+      filesystemmock("index.html", content)
+      SiteChecker.check(fs_test_path, @root)
+      SiteChecker.problems.should eql({fs_test_path => ["/a.png (404 Not Found)"]})
+    end
+
+    it "should be able to handle anchors in other files" do
+      content = "<html><a href=\"/other#goto\">goto</a>text<a id=\"goto\"></a></html>"
+      content2 = "<html><a id=\"goto\">goto</a>"
+      filesystemmock("index.html", content)
+      filesystemmock("other/index.html", content2)
+      SiteChecker.check(fs_test_path, @root)
+      SiteChecker.problems.should be_empty
+    end
+  end
+end

data/spec/site_checker/io/content_from_file_system_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+describe SiteChecker::IO::ContentFromFileSystem do
+  context "#get" do
+    before(:each) do
+      @root = "/home/test/web/public"
+      @link = SiteChecker::Link.create({:url => "link", :kind => :page, :location => :local})
+      @file = mock(File)
+      @content = mock()
+      @content_reader = SiteChecker::IO::ContentFromFileSystem.new(false, @root)
+    end
+
+    it "should return the content of a link using the local index.html" do
+      File.should_receive(:open).with("#{@root}/#{@link.url}/index.html") {@file}
+      @file.should_receive(:read) {@content}
+      @content_reader.get(@link).should eql(@content)
+    end
+
+    it "should return the content of a link which points to a real .html file" do
+      @link.url = "/about.html"
+      File.should_receive(:open).with("#{@root}/about.html") {@file}
+      @file.should_receive(:read) {@content}
+      @content_reader.get(@link).should eql(@content)
+    end
+
+    it "should return the content of a link with anchor" do
+      @link.url = "/about#something"
+      File.should_receive(:open).with("#{@root}/about/index.html") {@file}
+      @file.should_receive(:read) {@content}
+      @content_reader.get(@link).should eql(@content)
+    end
+
+    it "should raise error if the link is broken" do
+      File.should_receive(:open).with("#{@root}/#{@link.url}/index.html").and_raise(Errno::ENOENT)
+      expect {@content_reader.get(@link)}.to raise_error(RuntimeError, "(404 Not Found)")
+    end
+
+    it "should check the existence of a local image" do
+      @link.kind = :image
+      @link.url = "img/image1"
+      File.should_receive(:open).with("#{@root}/#{@link.url}") {@file}
+      @file.should_not_receive(:read)
+      @content_reader.get(@link).should
+    end
+
+    it "should not open a remote reference if opt-out" do
+      @link.location = :remote
+      File.should_not_receive(:open)
+      @content_reader.get(@link)
+    end
+
+    it "should open a remote reference if opt-in" do
+      @content_reader = SiteChecker::IO::ContentFromFileSystem.new(true, @root)
+      @link.location = :remote
+      @link.url = "http://example.org"
+      File.should_not_receive(:open)
+      @content_reader.should_receive(:open)
+      @content_reader.get(@link)
+    end
+  end
+end

data/spec/site_checker/io/content_from_web_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+require_relative 'io_spec_helper'
+
+describe SiteChecker::IO::ContentFromWeb do
+	include IoSpecHelper
+	context "#get" do
+		before(:each) do
+			@root = "http://localhost:4000"
+			@link = SiteChecker::Link.create({:url => "link", :kind => :page, :location => :local})
+			@content = mock()
+			@content_reader = SiteChecker::IO::ContentFromWeb.new(false, @root)
+		end
+
+		it "should return the content of a link" do
+      @content_reader.should_receive(:open).with(URI("#{@root}/#{@link.url}")).and_return(@content)
+      @content_reader.get(@link).should eql(@content)
+		end
+
+		it "should raise error if the link is broken" do
+			@content_reader.should_receive(:open).with(URI("#{@root}/#{@link.url}")).
+				and_raise(OpenURI::HTTPError.new("404 Not Found", nil))
+      expect {@content_reader.get(@link)}.to raise_error(RuntimeError, "(404 Not Found)")
+		end
+
+		it "should check the existence of an image" do
+			@link.kind = :image
+			@link.url = "img/image1"
+			@content_reader.should_receive(:open).with(URI("#{@root}/#{@link.url}"))
+			@content_reader.get(@link)
+		end
+
+		it "should not open a remote reference if opt-out" do
+			@link.location = :remote
+      @content_reader.should_not_receive(:open).with(URI("#{@root}/#{@link.url}"))
+      @content_reader.get(@link)
+		end
+
+		it "should open a remote reference if opt-in" do
+			@content_reader = SiteChecker::IO::ContentFromWeb.new(true, @root)
+			@link.location = :remote
+			@link.url = "http://example.org"
+      @content_reader.should_receive(:open).with(URI(@link.url))
+      @content_reader.get(@link)
+		end
+	end
+end