site_checker 0.2.1 → 0.3.0

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.8.7
+  - 1.9.3

data/History.md

@@ -1,3 +1,13 @@
+## [v0.3.0](https://github.com/ZsoltFabok/site_checker/compare/v0.2.1...v0.3.0)
+### Notes
+* the root argument is no longer mandatory (Zsolt Fabok)
+
+### Other Changes
+* you can use site_checker from the command line. Type site_checker -h for more details (Zsolt Fabok)
+
+### Fixes
+* the library is ruby 1.8 compatible (Zsolt Fabok)
+
 ## [v0.2.1](https://github.com/ZsoltFabok/site_checker/compare/v0.2.0...v0.2.1)
 ### Other Changes
 * less strict gem dependencies (Zsolt Fabok)

data/README.md

@@ -9,7 +9,7 @@ Site Checker is a simple ruby gem, which helps you check the integrity of your w
 
 ### Usage
 
-#### Default
+#### In Test Code
 
 First, you have to load the `site_checker` by adding this line to the file where you would like to use it:
 
@@ -37,13 +37,13 @@ In case you don't want to use a DSL like API you can still do the following:
     puts SiteChecker.local_images.inspect
     puts SiteChecker.problems.inspect
 
-### Using on Generated Content
+##### Using on Generated Content
 If you have a static website (e.g. generated by [octopress](https://github.com/imathis/octopress)) you can tell `site_checker` to use folders from the file system. With this approach, you don't need a webserver for verifying your website:
 
     check_site("./public", "./public")
     puts collected_problems.inspect
 
-### Configuration
+##### Configuration
 You can instruct `site_checker` to ignore certain links:
 
     SiteChecker.configure do |config|
@@ -62,7 +62,7 @@ Too deep recursive calls may be expensive, so you can configure the maximum dept
       config.max_recursion_depth = 3
     end
 
-### Examples
+##### Examples
 Make sure that there are no local dead links on the website (I'm using [rspec](https://github.com/rspec/rspec) syntax):
 
     before(:each) do
@@ -93,6 +93,24 @@ Check that all the local pages can be reached with maximum two steps:
       collected_local_pages.size.should eq @number_of_local_pages
     end
 
+#### Command line
+From version 0.3.0 the site checker can be used from the command line as well. Here is the list of the available options:
+
+    ~ % site_checker -h
+    Visits the <site_url> and prints out the list of those URLs which cannot be found
+
+    Usage: site_checker [options] <site_url>
+    -e, --visit-external-references  Visit external references (may take a bit longer)
+    -m, --max-recursion-depth N      Set the depth of the recursion
+    -r, --root URL                   The root URL of the path
+    -i, --ignore URL                 Ignore the provided URL (can be applied several times)
+    -p, --print-local-pages          Prints the list of the URLs of the collected local pages
+    -x, --print-remote-pages         Prints the list of the URLs of the collected remote pages
+    -y, --print-local-images         Prints the list of the URLs of the collected local images
+    -z, --print-remote-images        Prints the list of the URLs of the collected remote images
+    -h, --help                       Show a short description and this message
+    -v, --version                    Show version
+
 ### Troubleshooting
 #### undefined method 'new' for SiteChecker:Module
 This error occurs when the test code calls v0.1.1 methods, but a newer version of the gem has already been installed. Update your test code following the examples above.

data/bin/site_checker

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require "site_checker"
+SiteChecker::Cli.new.start

data/lib/site_checker/cli/cli.rb

@@ -0,0 +1,126 @@
+require 'optparse'
+
+module SiteChecker
+  class Cli
+    def start
+      begin
+        options = option_parser
+        configure_site_checker(options)
+        check_site(ARGV[0], options[:root])
+
+        print_problems_if_any
+
+        print(collected_local_pages, "Collected local pages:", options[:print_local_pages])
+        print(collected_remote_pages, "Collected remote pages:", options[:print_remote_pages])
+        print(collected_local_images, "Collected local images:", options[:print_local_images])
+        print(collected_remote_images, "Collected remote images:", options[:print_remote_images])
+
+      rescue Interrupt
+        puts "Error: Interrupted"
+      rescue SystemExit
+        puts
+      rescue Exception => e
+        puts "Error: #{e.message}"
+        puts
+      end
+    end
+
+    private
+    def option_parser
+      options, optparse = create_parser
+
+      begin
+        optparse.parse!
+        if ARGV.size != 1
+          raise OptionParser::MissingArgument.new("<site_url>")
+        end
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument, OptionParser::InvalidArgument
+        message = $!.to_s + "\n\n" + optparse.to_s
+        raise Exception.new(message)
+      end
+      options
+    end
+
+    def print(list, message, enabled)
+      if !list.empty? && enabled
+        puts message
+        list.sort.each do |entry|
+          puts "  #{entry}"
+        end
+      end
+    end
+
+    def print_problems_if_any
+      if not collected_problems.empty?
+        puts "Collected problems:"
+        collected_problems.keys.sort.each do |parent|
+          puts "  #{parent}"
+          collected_problems[parent].sort.each do |url|
+            puts "    #{url}"
+          end
+        end
+      end
+    end
+
+    def configure_site_checker(options)
+      SiteChecker.configure do |config|
+        config.ignore_list = options[:ignore_list] if options[:ignore_list]
+        config.visit_references = options[:visit_references] if options[:visit_references]
+        config.max_recursion_depth = options[:max_recursion_depth] if options[:max_recursion_depth]
+      end
+    end
+
+    def create_parser
+      options = {}
+      optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: site_checker [options] <site_url>"
+
+        opts.on("-e", "--visit-external-references", "Visit external references (may take a bit longer)") do |opt|
+          options[:visit_references] = opt
+        end
+
+        opts.on("-m", "--max-recursion-depth N", Integer, "Set the depth of the recursion") do |opt|
+          options[:max_recursion_depth] = opt
+        end
+
+        opts.on("-r", "--root URL", "The root URL of the path") do |opt|
+          options[:root] = opt
+        end
+
+        opts.on("-i", "--ignore URL", "Ignore the provided URL (can be applied several times)") do |opt|
+          options[:ignore_list] ||= []
+          options[:ignore_list] << opt
+        end
+
+        opts.on("-p","--print-local-pages", "Prints the list of the URLs of the collected local pages") do |opt|
+          options[:print_local_pages] = opt
+        end
+
+        opts.on("-x", "--print-remote-pages", "Prints the list of the URLs of the collected remote pages") do |opt|
+          options[:print_remote_pages] = opt
+        end
+
+        opts.on("-y", "--print-local-images", "Prints the list of the URLs of the collected local images") do |opt|
+          options[:print_local_images] = opt
+        end
+
+        opts.on("-z", "--print-remote-images", "Prints the list of the URLs of the collected remote images") do |opt|
+          options[:print_remote_images] = opt
+        end
+
+        opts.on_tail("-h", "--help", "Show a short description and this message") do
+          puts "Visits the <site_url> and prints out the list of those URLs which cannot be found"
+          puts
+          puts opts
+          exit
+        end
+
+        opts.on_tail("-v", "--version", "Show version") do
+          puts SiteChecker::VERSION
+          exit
+        end
+      end
+      [options, optparse]
+    end
+  end
+end

data/lib/site_checker/link.rb

@@ -1,13 +1,13 @@
 module SiteChecker
   class Link
-  	attr_accessor :url
+    attr_accessor :url
     attr_accessor :modified_url
-  	attr_accessor :parent_url
-  	attr_accessor :kind
-  	attr_accessor :location
-  	attr_accessor :problem
+    attr_accessor :parent_url
+    attr_accessor :kind
+    attr_accessor :location
+    attr_accessor :problem
 
-  	def eql?(other)
+    def eql?(other)
       @modified_url.eql? other.modified_url
     end
 
@@ -20,13 +20,13 @@ module SiteChecker
     end
 
     def self.create(attrs)
-    	link = Link.new
-    	attrs.each do |key, value|
-    		if self.instance_methods.include?("#{key}=".to_sym)
-    			eval("link.#{key}=value")
-    		end
-    	end
-    	link
+      link = Link.new
+      attrs.each do |key, value|
+        if self.instance_methods.map{|m| m.to_s}.include?("#{key}=")
+          eval("link.#{key}=value")
+        end
+      end
+      link
     end
 
     def parent_url=(parent_url)
@@ -40,19 +40,19 @@ module SiteChecker
     end
 
     def has_problem?
-    	@problem != nil
+      @problem != nil
     end
 
     def local_page?
-    	@location == :local && @kind == :page
+      @location == :local && @kind == :page
     end
 
     def local_image?
-    	@location == :local && @kind == :image
+      @location == :local && @kind == :image
     end
 
     def anchor?
-    	@kind == :anchor
+      @kind == :anchor
     end
 
     def anchor_ref?

data/lib/site_checker/link_collector.rb

@@ -9,10 +9,10 @@ module SiteChecker
       @max_recursion_depth ||= -1
     end
 
-    def check(url, root)
+    def check(url, root=nil)
       @links = {}
       @recursion_depth = 0
-      @root = root
+      @root = figure_out_root(url,root)
 
       @content_reader = get_content_reader
 
@@ -50,6 +50,18 @@ module SiteChecker
     end
 
     private
+    def figure_out_root(url, root)
+      unless root
+        url_uri = URI(url)
+        if url_uri.absolute?
+          root = "#{url_uri.scheme}://#{url_uri.host}"
+        else
+          root = url
+        end
+      end
+      root
+    end
+
     def get_content_reader
       if URI(@root).absolute?
         SiteChecker::IO::ContentFromWeb.new(@visit_references, @root)

data/lib/site_checker/version.rb

@@ -0,0 +1,3 @@
+module SiteChecker
+  VERSION = '0.3.0'
+end

data/lib/site_checker.rb

@@ -7,6 +7,8 @@ require 'site_checker/parse/page'
 require 'site_checker/link'
 require 'site_checker/link_collector'
 require 'site_checker/dsl'
+require 'site_checker/version'
+require 'site_checker/cli/cli'
 
 module SiteChecker
   class << self
@@ -44,9 +46,9 @@ module SiteChecker
     # Recursively visits the provided url looking for reference problems.
     #
     # @param [String] url where the processing starts
-    # @param [String] root the root URL of the site
+    # @param [String] root (optional) the root URL of the site. If not provided then the method will use the url to figure it out.
     #
-    def check(url, root)
+    def check(url, root=nil)
       create_instance
       @link_collector.check(url, root)
     end

data/site_checker.gemspec

@@ -1,7 +1,13 @@
 # -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'site_checker/version'
+
 Gem::Specification.new do |s|
   s.name        = 'site_checker'
-  s.version     = '0.2.1'
+  s.version     = SiteChecker::VERSION
   s.date        = '2012-12-31'
   s.summary     = "site_checker-#{s.version}"
   s.description = "A simple tool for checking references on your website"

data/spec/integration_spec.rb

@@ -188,4 +188,34 @@ describe "Integration" do
       SiteChecker.problems.should be_empty
     end
   end
+
+  describe "without root argument" do
+    before(:each) do
+      @test_url = "http://localhost:4000"
+    end
+
+    it "should check the link to an external page" do
+      content = "<html>text<a href=\"http://external.org/\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("http://external.org", 200, "")
+      SiteChecker.check(@test_url)
+      SiteChecker.remote_pages.should eql(["http://external.org/" ])
+      SiteChecker.problems.should be_empty
+    end
+
+    it "should report a problem for a local page with absolute path" do
+      content = "<html>text<a href=\"#{@test_url}/another\"/></html>"
+      webmock(@test_url, 200, content)
+      webmock("#{@test_url}/another", 200, "")
+      SiteChecker.check(@test_url)
+      SiteChecker.problems.should eql({@test_url => ["#{@test_url}/another (absolute path)"]})
+    end
+
+    it "should report a problem when the local image cannot be found" do
+      content = "<html>text<img src=\"/a.png\"/></html>"
+      filesystemmock("index.html", content)
+      SiteChecker.check(fs_test_path)
+      SiteChecker.problems.should eql({fs_test_path => ["/a.png (404 Not Found)"]})
+    end
+  end
 end

data/spec/site_checker/cli/cli_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+require 'site_checker/cli/cli_spec_helper'
+
+describe "CLI" do
+  include CliSpecHelper
+
+  before(:each) do
+    @command = File.expand_path('../../../bin/site_checker', File.dirname(__FILE__))
+    clean_fs_test_path
+  end
+
+  context "argument check" do
+    it "help option should print a description and the list of available options" do
+      description = "Visits the <site_url> and prints out the list of those URLs which cannot be found\n\n"
+      output = exec(@command, "-h")
+      output2 = exec(@command, "--help")
+      output.should eql(output2)
+      output.should eql(description + option_list)
+    end
+
+    it "version option should print the current version" do
+      output = exec(@command, "-v")
+      output2 = exec(@command, "--version")
+      output.should eql(output2)
+      output.should eql(SiteChecker::VERSION + "\n")
+    end
+
+    it "missing number after the max-recursion-depth option should print error and the list of available options" do
+      message = "Error: missing argument: --max-recursion-depth\n\n"
+      output = exec(@command, "--max-recursion-depth")
+      output.should eql(message + option_list)
+    end
+
+    it "missing site_url should print error message" do
+      message = "Error: missing argument: <site_url>\n\n"
+      output = exec(@command, "")
+      output.should eql(message + option_list)
+    end
+  end
+
+  context "execution" do
+    it "should run a basic check without any problem" do
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      filesystemmock("index.html", content)
+      filesystemmock("/one-level-down/index.html", content)
+
+      expected = "Collected local pages:\n  /one-level-down\n  test_data_public"
+
+      output = exec(@command, "--print-local-pages #{fs_test_path}")
+      output.should eql(expected)
+    end
+
+    it "should print out the collected problems" do
+      content = "<html>text<a href=\"/one-level-down\"/></html>"
+      filesystemmock("index.html", content)
+      filesystemmock("/two-levels-down/index.html", content)
+
+      expected = "Collected problems:\n  test_data_public\n    /one-level-down (404 Not Found)"
+
+      output = exec(@command, "#{fs_test_path}")
+      output.should eql(expected)
+    end
+
+    it "should ignore all the provided URLs" do
+      content = "<html>text<a href=\"/one-level-down\"/><a href=\"/two-levels-down\"/></html>"
+      filesystemmock("index.html", content)
+
+      expected = "Collected local pages:\n  test_data_public"
+
+      output = exec(@command, "--print-local-pages --ignore /one-level-down --ignore /two-levels-down #{fs_test_path}")
+      output.should eql(expected)
+    end
+  end
+end

data/spec/site_checker/cli/cli_spec_helper.rb

@@ -0,0 +1,25 @@
+require 'site_checker/io/io_spec_helper'
+require 'open3'
+
+module CliSpecHelper
+  include IoSpecHelper
+
+  def exec(command, arguments)
+  	stdin, stdout, stderr = Open3.popen3("#{command} #{arguments}")
+		stdout.readlines.map {|line| line.chomp}.join("\n")
+	end
+
+	def option_list
+		"Usage: site_checker [options] <site_url>\n" +
+		"    -e, --visit-external-references  Visit external references (may take a bit longer)\n" +
+		"    -m, --max-recursion-depth N      Set the depth of the recursion\n" +
+		"    -r, --root URL                   The root URL of the path\n" +
+		"    -i, --ignore URL                 Ignore the provided URL (can be applied several times)\n" +
+		"    -p, --print-local-pages          Prints the list of the URLs of the collected local pages\n" +
+		"    -x, --print-remote-pages         Prints the list of the URLs of the collected remote pages\n" +
+		"    -y, --print-local-images         Prints the list of the URLs of the collected local images\n" +
+		"    -z, --print-remote-images        Prints the list of the URLs of the collected remote images\n" +
+		"    -h, --help                       Show a short description and this message\n" +
+		"    -v, --version                    Show version\n"
+	end
+end

data/spec/site_checker/io/io_spec_helper.rb

@@ -1,22 +1,22 @@
 module IoSpecHelper
-	def webmock(uri, status, content)
-	stub_request(:get, uri).
-    with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
-    to_return(:status => status, :body => content)
-	end
+  def webmock(uri, status, content)
+    stub_request(:get, uri).
+      with(:headers => {'Accept'=>'*/*'}).\
+      to_return(:status => status, :body => content)
+  end
 
-	def filesystemmock(uri, content)
-		FileUtils.mkdir_p(File.dirname(File.join(fs_test_path, uri)))
-		File.open(File.join(fs_test_path, uri), "w") do |f|
-			f.write(content)
-		end
-	end
+  def filesystemmock(uri, content)
+    FileUtils.mkdir_p(File.dirname(File.join(fs_test_path, uri)))
+    File.open(File.join(fs_test_path, uri), "w") do |f|
+      f.write(content)
+    end
+  end
 
-	def clean_fs_test_path
-		FileUtils.rm_rf(fs_test_path)
-	end
+  def clean_fs_test_path
+    FileUtils.rm_rf(fs_test_path)
+  end
 
-	def fs_test_path
-		"test_data_public"
-	end
+  def fs_test_path
+    "test_data_public"
+  end
 end

data/spec/spec_helper.rb

@@ -1,10 +1,16 @@
 require 'rspec'
-require 'debugger'
 require 'webmock/rspec'
 
+begin
+  require "debugger"
+rescue LoadError
+  # most probably using 1.8
+  require "ruby-debug"
+end
+
 require File.expand_path('../../lib/site_checker', __FILE__)
 
 # common
 def create_link(url)
-	SiteChecker::Link.create({:url => url})
+  SiteChecker::Link.create({:url => url})
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_checker
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -93,27 +93,34 @@ dependencies:
         version: 1.5.6
 description: A simple tool for checking references on your website
 email: me@zsoltfabok.com
-executables: []
+executables:
+- site_checker
 extensions: []
 extra_rdoc_files: []
 files:
 - .rbenv-version
 - .rspec
+- .travis.yml
 - History.md
 - LICENSE
 - README.md
+- bin/site_checker
 - gem_tasks/rspec.rake
 - gem_tasks/yard.rake
 - lib/site_checker.rb
+- lib/site_checker/cli/cli.rb
 - lib/site_checker/dsl.rb
 - lib/site_checker/io/content_from_file_system.rb
 - lib/site_checker/io/content_from_web.rb
 - lib/site_checker/link.rb
 - lib/site_checker/link_collector.rb
 - lib/site_checker/parse/page.rb
+- lib/site_checker/version.rb
 - site_checker.gemspec
 - spec/dsl_spec.rb
 - spec/integration_spec.rb
+- spec/site_checker/cli/cli_spec.rb
+- spec/site_checker/cli/cli_spec_helper.rb
 - spec/site_checker/io/content_from_file_system_spec.rb
 - spec/site_checker/io/content_from_web_spec.rb
 - spec/site_checker/io/io_spec_helper.rb
@@ -137,7 +144,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 321001871174332709
+      hash: -2225611601008227703
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -146,16 +153,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 321001871174332709
+      hash: -2225611601008227703
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: site_checker-0.2.1
+summary: site_checker-0.3.0
 test_files:
 - spec/dsl_spec.rb
 - spec/integration_spec.rb
+- spec/site_checker/cli/cli_spec.rb
+- spec/site_checker/cli/cli_spec_helper.rb
 - spec/site_checker/io/content_from_file_system_spec.rb
 - spec/site_checker/io/content_from_web_spec.rb
 - spec/site_checker/io/io_spec_helper.rb