utterson 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 535d62e57f06b6f95ca5948cf2f235ff41e0dc05
-  data.tar.gz: 1e8098ec08f238c880596326d689d9272855793e
+  metadata.gz: b284cf9e9272e1093bd7c4172ef0036750de59e8
+  data.tar.gz: 512987cf376f785eb0621c7e374a002c3385cdb8
 SHA512:
-  metadata.gz: dcbfe2cb045061aedae22640f0042529a934c9dce5ca5bc59e7c1e6ca1419f7d04aadf28581524a15cc7b4ef63954c4d105ef6e2862e09bdc495fc14d3fce193
-  data.tar.gz: 2f11619c7568d1bf885493eb384aafc3a540ef8de6a6e109734bb04ece5027dbf6fc5016dab6c0fe872e20f0b5f9c991329deea6813a7e8a7470cd0f3eaa3864
+  metadata.gz: e1aec6ceb0f9748d1f5e378592159e322335214a6f96987bd253e38f2c4c42b6fe9a6c52791a0b9402228e71b5653bd9dfa0760360f3bab733e3a439eeb520c3
+  data.tar.gz: 64d65499f25e4c614995eced88b5a052c4623dbd8bbba7024abdfd512b696ecbe4d124b919eca19775e78ffaf2961e8281b2d17147732b5dbfaa9793e84131e4

data/bin/utterson

@@ -9,4 +9,4 @@ opts = Trollop::options do
   opt :root, "Root directory for the site if it differs from target dir", type: :string
 end
 
-Utterson.new(opts.merge(dir: ARGV[0])).check
+Utterson::Base.new(opts.merge(dir: ARGV[0])).check

data/lib/utterson.rb

@@ -1,105 +1,2 @@
-require 'nokogiri'
-
-require 'net/http'
-require 'timeout'
-
-class Utterson
-  attr_reader :errors
-
-  def initialize(opts={})
-    @dir = opts[:dir] || './'
-    @root = opts[:root] || @dir
-    @errors = {}
-    @checked_urls = {}
-    @stats = {errors: 0, files: 0, urls: 0}
-  end
-
-  def check
-    Dir.glob(File.join(@dir, '**/*.{html,htm}')) do |f|
-      @stats[:files] += 1
-      puts "Checking #{f}"
-      collect_uris_from(f).each do |u|
-        @stats[:urls] += 1
-        check_uri(u, f)
-      end
-    end
-    print_results
-  end
-
-  def collect_uris_from(f)
-    ret = []
-    doc = Nokogiri::HTML(File.read(f))
-    doc.traverse do |el|
-      ret << el['src'] unless el['src'].nil?
-      ret << el['href'] unless el['href'].nil?
-    end
-    ret
-  end
-
-  def check_uri(url, file)
-    return if @checked_urls[url]
-
-    if url =~ /^(https?:)?\/\//
-      check_remote_uri url, file
-    else
-      check_local_uri url, file
-    end
-    @checked_urls[url] = true
-  end
-
-  def check_remote_uri(url, file)
-    begin
-      uri = URI(url.gsub(/^\/\//, 'http://'))
-    rescue URI::InvalidURIError => e
-      return add_error(file, uri.to_s, e.message)
-    end
-    begin
-      response = Net::HTTP.start(uri.host, uri.port,
-                                 :use_ssl => uri.scheme == 'https') do |http|
-        p = uri.path.empty? ? "/" : uri.path
-        http.head(p)
-      end
-      if response.code =~ /^[^23]/
-        add_error(file, uri.to_s, response)
-      end
-    rescue Timeout::Error
-      add_error(file, uri.to_s, "Reading buffer timed out")
-    rescue Errno::ETIMEDOUT
-      add_error(file, uri.to_s, "Connection timed out")
-    rescue SocketError => e
-      add_error(file, uri.to_s, e.message)
-    end
-  end
-
-  def check_local_uri(url, file)
-    url.gsub!(/\?.*$/, '')
-    if url =~ /^\//
-      path = File.expand_path(".#{url}", @root)
-    else
-      path = File.expand_path(url, File.dirname(file))
-    end
-    add_error(file, url, "File not found") unless File.exists? path
-  end
-
-  def add_error(file, url, response)
-    @stats[:errors] += 1
-    @errors[file] = {} if @errors[file].nil?
-    @errors[file][url] = response
-  end
-
-  def print_results
-    @errors.each do |file, info|
-      puts file
-      info.each do |url, response|
-        s = response.respond_to?(:code) ? "HTTP #{response.code}" : response
-        puts "\t#{url}\n\t\t#{s}"
-      end
-    end
-    if @stats[:errors] == 0
-      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked."
-    else
-      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked and #{@stats[:errors]} errors found."
-    end
-  end
-
-end
+require 'utterson/base'
+require 'utterson/version'

data/lib/utterson/base.rb

@@ -0,0 +1,49 @@
+require 'utterson/html_check'
+
+module Utterson
+  class Base
+    attr_reader :errors
+
+    def initialize(opts={})
+      @dir = opts[:dir] || './'
+      @root = opts[:root] || @dir
+      @errors = {}
+      @checked_urls = {}
+      @stats = {errors: 0, files: 0, urls: 0}
+    end
+
+    def check
+      threads = []
+      Dir.glob(File.join(@dir, '**/*.{html,htm}')) do |f|
+        @stats[:files] += 1
+        puts "Checking file #{f}"
+        c = HtmlCheck.new(file: f, root: @root)
+        c.when_done do |r|
+          @stats[:urls] = r[:urls]
+          @errors.merge! r[:errors]
+          puts "Check done with #{f}"
+        end
+        threads << c.run
+      end
+      threads.each {|t| t.join}
+      print_results
+    end
+
+    def print_results
+      count = 0
+      @errors.each do |file, info|
+        puts file
+        info.each do |url, response|
+          s = response.respond_to?(:code) ? "HTTP #{response.code}" : response
+          puts "\t#{url}\n\t\t#{s}"
+          count += 1
+        end
+      end
+      if count == 0
+        puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked."
+      else
+        puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked and #{count} errors found."
+      end
+    end
+  end
+end

data/lib/utterson/html_check.rb

@@ -0,0 +1,98 @@
+require 'nokogiri'
+
+require 'net/http'
+require 'timeout'
+require 'thread'
+
+module Utterson
+  class HtmlCheck
+    attr_reader :errors
+
+    @@semaphore = Mutex.new
+    @@checked_urls = {}
+
+    def initialize(opts={})
+      @file = opts[:file]
+      @root = opts[:root]
+      @errors = {}
+    end
+
+    def when_done(&handler)
+      @result_handler = handler
+    end
+
+    def run
+      Thread.new do
+        collect_uris_from(@file).each do |u|
+          check_uri(u, @file)
+        end
+        unless @result_handler.nil?
+          @@semaphore.synchronize do
+            @result_handler.call(errors: @errors, urls: @@checked_urls.count)
+          end
+        end
+      end
+    end
+
+    def collect_uris_from(f)
+      ret = []
+      doc = Nokogiri::HTML(File.read(f))
+      doc.traverse do |el|
+        ret << el['src'] unless el['src'].nil?
+        ret << el['href'] unless el['href'].nil?
+      end
+      ret
+    end
+
+    def check_uri(url, file)
+      @@semaphore.synchronize do
+        if @@checked_urls[url]
+          return
+        else
+          @@checked_urls[url] = true
+        end
+      end
+
+      if url =~ /^(https?:)?\/\//
+        check_remote_uri url, file
+      else
+        check_local_uri url, file
+      end
+    end
+
+    def check_remote_uri(url, file)
+      begin
+        uri = URI(url.gsub(/^\/\//, 'http://'))
+      rescue URI::InvalidURIError => e
+        return add_error(file, uri.to_s, e.message)
+      end
+      begin
+        response = Net::HTTP.start(uri.host, uri.port,
+                                   :use_ssl => uri.scheme == 'https') do |http|
+          p = uri.path.empty? ? "/" : uri.path
+          http.head(p)
+        end
+        if response.code =~ /^[^23]/
+          add_error(file, uri.to_s, response)
+        end
+      rescue => e
+        add_error(file, uri.to_s, e.message)
+      end
+    end
+
+    def check_local_uri(url, file)
+      url.gsub!(/\?.*$/, '')
+      if url =~ /^\//
+        path = File.expand_path(".#{url}", @root)
+      else
+        path = File.expand_path(url, File.dirname(file))
+      end
+      add_error(file, url, "File not found") unless File.exists? path
+    end
+
+    def add_error(file, url, response)
+      @errors[file] = {} if @errors[file].nil?
+      @errors[file][url] = response
+    end
+  end
+end

data/lib/utterson/version.rb

@@ -0,0 +1,3 @@
+module Utterson
+  VERSION = '0.2.0'
+end

data/spec/utterson/base_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+module Utterson
+  describe Base do
+    it "should go through all htm and html files in target dir" do
+      dir = "spec/fixtures/dir-structure"
+      u = Base.new(dir: dir)
+      HtmlCheck.stub(:new) {double(when_done: {}, run: double(join: {}))}
+
+      ["spec/fixtures/dir-structure/1.htm",
+       "spec/fixtures/dir-structure/2.html",
+       "spec/fixtures/dir-structure/a/3.htm",
+       "spec/fixtures/dir-structure/a/b/4.html"].each do |file|
+        HtmlCheck.should_receive(:new).with(file: file, root: dir)
+      end
+
+      u.check
+    end
+
+    describe "#print_results" do
+      it "should output only basic stats if no errors" do
+        u = Base.new(dir: "spec/fixtures/dir-structure")
+        output = capture_stdout do
+          u.check
+        end
+        output.should match(/4 files with 0 urls checked/)
+      end
+
+      it "should output error information" do
+        stub_request(:head, "http://example.com/").
+          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+          to_return(:status => 404, :body => "", :headers => {})
+        u = Base.new(dir: "spec/fixtures")
+        output = capture_stdout do
+          u.check
+        end
+        output.should match("spec/fixtures/sample.html\n\tstyle.css\n\t\tFile not found")
+        output.should match("script.js\n\t\tFile not found")
+        output.should match("image.jpg\n\t\tFile not found")
+        output.should match("http://example.com\n\t\tHTTP 404")
+        output.should match("5 files with 4 urls checked and 4 errors found")
+      end
+    end
+  end
+end

data/spec/utterson/html_check_spec.rb

@@ -0,0 +1,156 @@
+require 'spec_helper'
+
+module Utterson
+  describe HtmlCheck do
+    before(:each) do
+      HtmlCheck.class_variable_set(:@@checked_urls, {})
+    end
+
+    let(:sample_file) {"spec/fixtures/sample.html"}
+
+    it "should check all urls which are found" do
+      h = HtmlCheck.new(dir: "spec/fixtures", file: "spec/fixtures/sample.html")
+      h.stub(:check_uri) {}
+      h.should_receive(:check_uri).exactly(4).times
+      h.run.join
+    end
+
+    it "should find all uris from sample document" do
+      h = HtmlCheck.new(file: sample_file)
+      uris = h.collect_uris_from(sample_file)
+      uris.should include("script.js")
+      uris.should include("style.css")
+      uris.should include("http://example.com")
+      uris.should include("image.jpg")
+    end
+
+    describe "#check_uri" do
+      let(:h) {HtmlCheck.new}
+      let(:html_file) {"file.html"}
+
+      it "should check same url only once" do
+        url = "http://example.com"
+        h.stub(:check_remote_uri) {}
+        h.should_receive(:check_remote_uri).once.with(url, html_file)
+        h.check_uri(url, html_file)
+        h.check_uri(url, html_file)
+      end
+
+      it "should use remote checking for http protocol" do
+        url = "http://example.com"
+        h.stub(:check_remote_uri) {}
+        h.should_receive(:check_remote_uri).with(url, html_file)
+        h.check_uri(url, html_file)
+      end
+
+      it "should use remote checking for https protocol" do
+        url = "https://example.com"
+        h.stub(:check_remote_uri) {}
+        h.should_receive(:check_remote_uri).with(url, html_file)
+        h.check_uri(url, html_file)
+      end
+
+      it "should use remote checking when only // is specified" do
+        url = "//example.com"
+        h.stub(:check_remote_uri) {}
+        h.should_receive(:check_remote_uri).with(url, html_file)
+        h.check_uri(url, html_file)
+      end
+
+      it "should use local checking for relative uris" do
+        url = "../file.html"
+        h.stub(:check_local_uri) {}
+        h.should_receive(:check_local_uri).with(url, html_file)
+        h.check_uri(url, html_file)
+      end
+    end
+
+    describe "#check_local_uri" do
+      let(:h) {HtmlCheck.new(root: "spec/fixtures/dir-structure")}
+      let(:html_file) {"spec/fixtures/dir-structure/1.htm"}
+
+      it "should not assign error info if file exists" do
+        h.check_local_uri("../sample.html", html_file)
+        h.errors.should be_empty
+      end
+
+      it "should assign error info if file doesn't exist" do
+        url = "../sample_not_found.html"
+        h.check_local_uri(url, html_file)
+        h.errors[html_file].should == {url => "File not found"}
+      end
+
+      it "should use root directory when urls start with /" do
+        h2 = HtmlCheck.new(file: html_file,
+                           root: "spec/fixtures")
+        h2.check_local_uri("/sample.html", html_file)
+        h2.errors.should be_empty
+      end
+
+      it "should handle target directory as root for urls starting with / if root is no available" do
+        h.check_local_uri("/2.html", html_file)
+        h.errors.should be_empty
+      end
+
+      it "should ignore query string when checking local files" do
+        h.check_local_uri("2.html?queryparam=value", html_file)
+        h.errors.should be_empty
+      end
+    end
+
+    describe "#check_remote_uri" do
+      let(:h) {HtmlCheck.new(file: "test.html")}
+      let(:html_file) {"test.html"}
+      let(:url) {"http://example.com/index.html"}
+
+      it "should not assign error info if request is successfull" do
+        stub_request(:head, url).
+          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+          to_return(:status => 200, :body => "", :headers => {})
+        h.check_remote_uri(url, html_file)
+        h.errors.should be_empty
+      end
+
+      it "should assign error info if there is error response" do
+        stub_request(:head, url).
+          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+          to_return(:status => 404, :body => "", :headers => {})
+        h.check_remote_uri(url, html_file)
+        puts h.errors.inspect
+        h.errors[html_file].should_not be_empty
+        h.errors[html_file][url].instance_of?(Net::HTTPNotFound).should be_true
+      end
+
+      it "should add error status from buffer timeouts" do
+        stub_request(:head, url).to_timeout
+        h.check_remote_uri(url, html_file)
+        h.errors.should_not be_empty
+      end
+
+      it "should add error status from connection timeouts" do
+        stub_request(:head, url).to_raise(Errno::ETIMEDOUT)
+        h.check_remote_uri(url, html_file)
+        h.errors.should_not be_empty
+      end
+
+      it "should add error status from 'No route to host' errors" do
+        stub_request(:head, url).to_raise(Errno::EHOSTUNREACH)
+        h.check_remote_uri(url, html_file)
+        h.errors.should_not be_empty
+      end
+
+      it "shoud add error status from name resolution errors" do
+        stub_request(:head, url).
+          to_raise(SocketError.new('getaddrinfo: Name or service not known'))
+        h.check_remote_uri(url, html_file)
+        h.errors.should_not be_empty
+      end
+
+      it "shoud add error status when invalid URI" do
+        URI.stub(:new).and_raise(URI::InvalidURIError)
+        h.check_remote_uri("http://invalid_uri", html_file)
+        h.errors.should_not be_empty
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: utterson
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Juhamatti Niemelä
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-06 00:00:00.000000000 Z
+date: 2013-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: trollop
@@ -103,15 +103,19 @@ extensions: []
 extra_rdoc_files: []
 files:
 - bin/utterson
+- lib/utterson/html_check.rb
+- lib/utterson/version.rb
+- lib/utterson/base.rb
 - lib/utterson.rb
 - README.md
+- spec/utterson/base_spec.rb
+- spec/utterson/html_check_spec.rb
 - spec/spec_helper.rb
 - spec/fixtures/dir-structure/a/3.htm
 - spec/fixtures/dir-structure/a/b/4.html
 - spec/fixtures/dir-structure/2.html
 - spec/fixtures/dir-structure/1.htm
 - spec/fixtures/sample.html
-- spec/utterson_spec.rb
 homepage: https://github.com/iiska/utterson
 licenses:
 - MIT
@@ -137,11 +141,12 @@ signing_key:
 specification_version: 4
 summary: Friendly HTML crawler and url checker
 test_files:
+- spec/utterson/base_spec.rb
+- spec/utterson/html_check_spec.rb
 - spec/spec_helper.rb
 - spec/fixtures/dir-structure/a/3.htm
 - spec/fixtures/dir-structure/a/b/4.html
 - spec/fixtures/dir-structure/2.html
 - spec/fixtures/dir-structure/1.htm
 - spec/fixtures/sample.html
-- spec/utterson_spec.rb
 has_rdoc: 

data/spec/utterson_spec.rb

@@ -1,174 +0,0 @@
-require 'spec_helper'
-
-describe Utterson do
-  it "should go through all htm and html files in target dir" do
-    u = Utterson.new(dir: "spec/fixtures/dir-structure")
-    u.stub(:collect_uris_from) {[]}
-
-    ["spec/fixtures/dir-structure/1.htm",
-     "spec/fixtures/dir-structure/2.html",
-     "spec/fixtures/dir-structure/a/3.htm",
-     "spec/fixtures/dir-structure/a/b/4.html"].each do |file|
-      u.should_receive(:collect_uris_from).with(file)
-    end
-
-    u.check
-  end
-
-  it "should check all urls which are found" do
-    u = Utterson.new(dir: "spec/fixtures")
-    u.stub(:check_uri) {}
-    u.should_receive(:check_uri).exactly(4).times
-
-    u.check
-  end
-
-  it "should find all uris from sample document" do
-    u = Utterson.new
-    uris = u.collect_uris_from("spec/fixtures/sample.html")
-    uris.should include("script.js")
-    uris.should include("style.css")
-    uris.should include("http://example.com")
-    uris.should include("image.jpg")
-  end
-
-  describe "#check_uri" do
-    let(:u) {Utterson.new}
-    let(:html_file) {"file.html"}
-
-    it "should use remote checking for http protocol" do
-      url = "http://example.com"
-      u.stub(:check_remote_uri) {}
-      u.should_receive(:check_remote_uri).with(url, html_file)
-      u.check_uri(url, html_file)
-    end
-
-    it "should use remote checking for https protocol" do
-      url = "https://example.com"
-      u.stub(:check_remote_uri) {}
-      u.should_receive(:check_remote_uri).with(url, html_file)
-      u.check_uri(url, html_file)
-    end
-
-    it "should use remote checking when only // is specified" do
-      url = "//example.com"
-      u.stub(:check_remote_uri) {}
-      u.should_receive(:check_remote_uri).with(url, html_file)
-      u.check_uri(url, html_file)
-    end
-
-    it "should use local checking for relative uris" do
-      url = "../file.html"
-      u.stub(:check_local_uri) {}
-      u.should_receive(:check_local_uri).with(url, html_file)
-      u.check_uri(url, html_file)
-    end
-  end
-
-  describe "#check_local_uri" do
-    let(:u) {Utterson.new(dir: "spec/fixtures/dir-structure")}
-    let(:html_file) {"spec/fixtures/dir-structure/1.htm"}
-
-    it "should not assign error info if file exists" do
-      u.check_local_uri("../sample.html", html_file)
-      u.errors.should be_empty
-    end
-
-    it "should assign error info if file doesn't exist" do
-      url = "../sample_not_found.html"
-      u.check_local_uri(url, html_file)
-      u.errors[html_file].should == {url => "File not found"}
-    end
-
-    it "should use root directory when urls start with /" do
-      u2 = Utterson.new(dir: "spec/fixtures/dir-structure",
-                        root: "spec/fixtures")
-      u2.check_local_uri("/sample.html", html_file)
-      u2.errors.should be_empty
-    end
-
-    it "should handle target directory as root for urls starting with / if root is no available" do
-      u.check_local_uri("/2.html", html_file)
-      u.errors.should be_empty
-    end
-
-    it "should ignore query string when checking local files" do
-      u.check_local_uri("2.html?queryparam=value", html_file)
-      u.errors.should be_empty
-    end
-  end
-
-  describe "#check_remote_uri" do
-    let(:u) {Utterson.new}
-    let(:html_file) {"test.html"}
-    let(:url) {"http://example.com/index.html"}
-
-    it "should not assign error info if request is successfull" do
-      stub_request(:head, url).
-        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
-        to_return(:status => 200, :body => "", :headers => {})
-      u.check_remote_uri(url, html_file)
-      u.errors.should be_empty
-    end
-
-    it "should assign error info if there is error response" do
-      stub_request(:head, url).
-        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
-        to_return(:status => 404, :body => "", :headers => {})
-      u.check_remote_uri(url, html_file)
-      puts u.errors.inspect
-      u.errors[html_file].should_not be_empty
-      u.errors[html_file][url].instance_of?(Net::HTTPNotFound).should be_true
-    end
-
-    it "should add error status from buffer timeouts" do
-      stub_request(:head, url).to_timeout
-      u.check_remote_uri(url, html_file)
-      u.errors.should_not be_empty
-    end
-
-    it "should add error status from connection timeouts" do
-      stub_request(:head, url).to_raise(Errno::ETIMEDOUT)
-      u.check_remote_uri(url, html_file)
-      u.errors.should_not be_empty
-    end
-
-    it "shoud add error status from name resolution errors" do
-      stub_request(:head, url).
-        to_raise(SocketError.new('getaddrinfo: Name or service not known'))
-      u.check_remote_uri(url, html_file)
-      u.errors.should_not be_empty
-    end
-
-    it "shoud add error status when invalid URI" do
-      URI.stub(:new).and_raise(URI::InvalidURIError)
-      u.check_remote_uri("http://invalid_uri", html_file)
-      u.errors.should_not be_empty
-    end
-  end
-
-  describe "#print_results" do
-    it "should output only basic stats if no errors" do
-      u = Utterson.new(dir: "spec/fixtures/dir-structure")
-      output = capture_stdout do
-        u.check
-      end
-      output.should match(/4 files with 0 urls checked/)
-    end
-
-    it "should output error information" do
-      stub_request(:head, "http://example.com/").
-        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
-        to_return(:status => 404, :body => "", :headers => {})
-      u = Utterson.new(dir: "spec/fixtures")
-      output = capture_stdout do
-        u.check
-      end
-      output.should match("spec/fixtures/sample.html\n\tstyle.css\n\t\tFile not found")
-      output.should match("script.js\n\t\tFile not found")
-      output.should match("image.jpg\n\t\tFile not found")
-      output.should match("http://example.com\n\t\tHTTP 404")
-      output.should match("5 files with 4 urls checked and 4 errors found")
-    end
-  end
-end