RubyGems - utterson - Versions diffs - 0.1.0 - Mend

utterson 0.1.0

Files changed (12) hide show

checksums.yaml +7 -0
data/README.md +8 -0
data/bin/utterson +12 -0
data/lib/utterson.rb +105 -0
data/spec/fixtures/dir-structure/1.htm +0 -0
data/spec/fixtures/dir-structure/2.html +0 -0
data/spec/fixtures/dir-structure/a/3.htm +0 -0
data/spec/fixtures/dir-structure/a/b/4.html +0 -0
data/spec/fixtures/sample.html +15 -0
data/spec/spec_helper.rb +20 -0
data/spec/utterson_spec.rb +164 -0
metadata +147 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef4a4e14d3f5b03451d25de3bbd09e7a977ec2b4
+  data.tar.gz: 5fb7fe1675f420b24267cc78e937bf98db2e4d8a
+SHA512:
+  metadata.gz: 3244214395c560032c66ca75d82fe73c319bf1ac527783bb263a9e1676f3910057fc83ae529ba4db99f4ccbc7691ed367d2c5fea9ec514b237f079ad965a04e5
+  data.tar.gz: cc7d65f5cf0689d10ed99a65d62bcc5a1ee82589f2eebd2aefba9a533f5cfc1861ba49b33bd8e6fb0107191fc035dbad07e274b01f9fa6acb82c4861a786ce14

data/README.md ADDED

@@ -0,0 +1,8 @@
+Simple utility to traverse directory of html files and check links in them.
+## Why the name?
+I developed this to help me checking links in my Jekyll powered blog
+and Mr. Utterson is the main character in the [Strange Case of Dr
+Jekyll and Mr
+Hyde](https://en.wikipedia.org/wiki/Strange_Case_of_Dr_Jekyll_and_Mr_Hyde).

data/bin/utterson ADDED

@@ -0,0 +1,12 @@
+#! /bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+require 'trollop'
+require 'utterson'
+opts = Trollop::options do
+  opt :root, "Root directory for the site if it differs from target dir", type: :string
+end
+Utterson.new(opts.merge(dir: ARGV[0])).check

data/lib/utterson.rb ADDED

@@ -0,0 +1,105 @@
+require 'nokogiri'
+require 'net/http'
+require 'timeout'
+class Utterson
+  attr_reader :errors
+  def initialize(opts={})
+    @dir = opts[:dir] || './'
+    @root = opts[:root] || @dir
+    @errors = {}
+    @checked_urls = {}
+    @stats = {errors: 0, files: 0, urls: 0}
+  end
+  def check
+    Dir.glob(File.join(@dir, '**/*.{html,htm}')) do |f|
+      @stats[:files] += 1
+      puts "Checking #{f}"
+      collect_uris_from(f).each do |u|
+        @stats[:urls] += 1
+        check_uri(u, f)
+      end
+    end
+    print_results
+  end
+  def collect_uris_from(f)
+    ret = []
+    doc = Nokogiri::HTML(File.read(f))
+    doc.traverse do |el|
+      ret << el['src'] unless el['src'].nil?
+      ret << el['href'] unless el['href'].nil?
+    end
+    ret
+  end
+  def check_uri(url, file)
+    return if @checked_urls[url]
+    if url =~ /^(https?:)?\/\//
+      check_remote_uri url, file
+    else
+      check_local_uri url, file
+    end
+    @checked_urls[url] = true
+  end
+  def check_remote_uri(url, file)
+    begin
+      uri = URI(url.gsub(/^\/\//, 'http://'))
+    rescue URI::InvalidURIError => e
+      return add_error(file, uri.to_s, e.message)
+    end
+    begin
+      response = Net::HTTP.start(uri.host, uri.port,
+                                 :use_ssl => uri.scheme == 'https') do |http|
+        p = uri.path.empty? ? "/" : uri.path
+        http.head(p)
+      end
+      if response.code =~ /^[^23]/
+        add_error(file, uri.to_s, response)
+      end
+    rescue Timeout::Error
+      add_error(file, uri.to_s, "Reading buffer timed out")
+    rescue Errno::ETIMEDOUT
+      add_error(file, uri.to_s, "Connection timed out")
+    rescue SocketError => e
+      add_error(file, uri.to_s, e.message)
+    end
+  end
+  def check_local_uri(url, file)
+    url.gsub!(/\?.*$/, '')
+    if url =~ /^\//
+      path = File.expand_path(".#{url}", @root)
+    else
+      path = File.expand_path(url, File.dirname(file))
+    end
+    add_error(file, url, "File not found") unless File.exists? path
+  end
+  def add_error(file, url, response)
+    @stats[:errors] += 1
+    @errors[file] = {} if @errors[file].nil?
+    @errors[file][url] = response
+  end
+  def print_results
+    @errors.each do |file, info|
+      puts file
+      info.each do |url, response|
+        s = response.respond_to?(:code) ? "HTTP #{response.code}" : response
+        puts "\t#{url}\n\t\t#{s}"
+      end
+    end
+    if @stats[:errors] == 0
+      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked."
+    else
+      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked and #{@stats[:errors]} errors found."
+    end
+  end
+end

data/spec/fixtures/dir-structure/1.htm ADDED

File without changes

data/spec/fixtures/dir-structure/2.html ADDED

File without changes

data/spec/fixtures/dir-structure/a/3.htm ADDED

File without changes

data/spec/fixtures/dir-structure/a/b/4.html ADDED

File without changes

data/spec/fixtures/sample.html ADDED

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Title</title>
+    <link type="text/css" rel="stylesheet" href="style.css" />
+    <script type="text/javascript" src="script.js"></script>
+  </head>
+  <body>
+    <h1>I'm a heading!</h1>
+    <p>I'm an <a href="http://example.com">example link</a>.</p>
+    <img src="image.jpg" />
+  </body>
+</html>

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,20 @@
+#require 'rubygems'
+require 'webmock/rspec'
+require 'simplecov'
+SimpleCov.start
+require 'utterson'
+require 'stringio'
+def capture_stdout &block
+  old_stdout = $stdout
+  fake_stdout = StringIO.new
+  $stdout = fake_stdout
+  block.call
+  fake_stdout.string
+ensure
+  $stdout = old_stdout
+end

data/spec/utterson_spec.rb ADDED

@@ -0,0 +1,164 @@
+require 'spec_helper'
+describe Utterson do
+  it "should go through all htm and html files in target dir" do
+    u = Utterson.new(dir: "spec/fixtures/dir-structure")
+    u.stub(:collect_uris_from) {[]}
+    ["spec/fixtures/dir-structure/1.htm",
+     "spec/fixtures/dir-structure/2.html",
+     "spec/fixtures/dir-structure/a/3.htm",
+     "spec/fixtures/dir-structure/a/b/4.html"].each do |file|
+      u.should_receive(:collect_uris_from).with(file)
+    end
+    u.check
+  end
+  it "should check all urls which are found" do
+    u = Utterson.new(dir: "spec/fixtures")
+    u.stub(:check_uri) {}
+    u.should_receive(:check_uri).exactly(4).times
+    u.check
+  end
+  it "should find all uris from sample document" do
+    u = Utterson.new
+    uris = u.collect_uris_from("spec/fixtures/sample.html")
+    uris.should include("script.js")
+    uris.should include("style.css")
+    uris.should include("http://example.com")
+    uris.should include("image.jpg")
+  end
+  describe "#check_uri" do
+    let(:u) {Utterson.new}
+    it "should use remote checking for http protocol" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("http://example.com", "file.html")
+      u.check_uri("http://example.com", "file.html")
+    end
+    it "should use remote checking for https protocol" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("https://example.com", "file.html")
+      u.check_uri("https://example.com", "file.html")
+    end
+    it "should use remote checking when only // is specified" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("//example.com", "file.html")
+      u.check_uri("//example.com", "file.html")
+    end
+    it "should use local checking for relative uris" do
+      u.stub(:check_local_uri) {}
+      u.should_receive(:check_local_uri).with("../file.html", "file.html")
+      u.check_uri("../file.html", "file.html")
+    end
+  end
+  describe "#check_local_uri" do
+    let(:u) {Utterson.new(dir: "spec/fixtures/dir-structure")}
+    it "should not assign error info if file exists" do
+      u.check_local_uri("../sample.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+    it "should assign error info if file doesn't exist" do
+      u.check_local_uri("../sample_not_found.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors["spec/fixtures/dir-structure/1.htm"].should == {"../sample_not_found.html" => "File not found"}
+    end
+    it "should use root directory when urls start with /" do
+      u2 = Utterson.new(dir: "spec/fixtures/dir-structure", root: "spec/fixtures")
+      u2.check_local_uri("/sample.html", "spec/fixtures/dir-structure/1.htm")
+      u2.errors.should be_empty
+    end
+    it "should handle target directory as root for urls starting with / if root is no available" do
+      u.check_local_uri("/2.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+    it "should ignore query string when checking local files" do
+      u.check_local_uri("2.html?queryparam=value", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+  end
+  describe "#check_remote_uri" do
+    let(:u) {Utterson.new}
+    it "should not assign error info if request is successfull" do
+      stub_request(:head, "http://example.com/index.html").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 200, :body => "", :headers => {})
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should be_empty
+    end
+    it "should assign error info if there is error response" do
+      stub_request(:head, "http://example.com/404.html").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 404, :body => "", :headers => {})
+      u.check_remote_uri("http://example.com/404.html", "test.html")
+      puts u.errors.inspect
+      u.errors["test.html"].should_not be_empty
+      u.errors["test.html"]["http://example.com/404.html"].instance_of?(Net::HTTPNotFound).should be_true
+    end
+    it "should add error status from buffer timeouts" do
+      stub_request(:head, "http://example.com/index.html").to_timeout
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "should add error status from connection timeouts" do
+      stub_request(:head, "http://example.com/index.html").to_raise(Errno::ETIMEDOUT)
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "shoud add error status from name resolution errors" do
+      stub_request(:head, "http://example.com/index.html").
+        to_raise(SocketError.new('getaddrinfo: Name or service not known'))
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "shoud add error status when invalid URI" do
+      URI.stub(:new).and_raise(URI::InvalidURIError)
+      u.check_remote_uri("http://invalid_uri", "test.html")
+      u.errors.should_not be_empty
+    end
+  end
+  describe "#print_results" do
+    it "should output only basic stats if no errors" do
+      u = Utterson.new(dir: "spec/fixtures/dir-structure")
+      output = capture_stdout do
+        u.check
+      end
+      output.should match(/4 files with 0 urls checked/)
+    end
+    it "should output error information" do
+      stub_request(:head, "http://example.com/").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 404, :body => "", :headers => {})
+      u = Utterson.new(dir: "spec/fixtures")
+      output = capture_stdout do
+        u.check
+      end
+      output.should match("spec/fixtures/sample.html\n\tstyle.css\n\t\tFile not found")
+      output.should match("script.js\n\t\tFile not found")
+      output.should match("image.jpg\n\t\tFile not found")
+      output.should match("http://example.com\n\t\tHTTP 404")
+      output.should match("5 files with 4 urls checked and 4 errors found")
+    end
+  end
+end

metadata ADDED

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: utterson
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Juhamatti Niemelä
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-11-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.15.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.15.2
+description: Traverses all HTML files from given directory and checks links found
+  in them.
+email: iiska@iki.fi
+executables:
+- utterson
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/utterson
+- lib/utterson.rb
+- README.md
+- spec/spec_helper.rb
+- spec/fixtures/dir-structure/a/3.htm
+- spec/fixtures/dir-structure/a/b/4.html
+- spec/fixtures/dir-structure/2.html
+- spec/fixtures/dir-structure/1.htm
+- spec/fixtures/sample.html
+- spec/utterson_spec.rb
+homepage: https://github.com/iiska/utterson
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ~>
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.0.3
+signing_key:
+specification_version: 4
+summary: Friendly HTML crawler and url checker
+test_files:
+- spec/spec_helper.rb
+- spec/fixtures/dir-structure/a/3.htm
+- spec/fixtures/dir-structure/a/b/4.html
+- spec/fixtures/dir-structure/2.html
+- spec/fixtures/dir-structure/1.htm
+- spec/fixtures/sample.html
+- spec/utterson_spec.rb
+has_rdoc: