RubyGems - utterson - Versions diffs - 0.1.0 - Mend

utterson 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (12) hide show

checksums.yaml +7 -0
data/README.md +8 -0
data/bin/utterson +12 -0
data/lib/utterson.rb +105 -0
data/spec/fixtures/dir-structure/1.htm +0 -0
data/spec/fixtures/dir-structure/2.html +0 -0
data/spec/fixtures/dir-structure/a/3.htm +0 -0
data/spec/fixtures/dir-structure/a/b/4.html +0 -0
data/spec/fixtures/sample.html +15 -0
data/spec/spec_helper.rb +20 -0
data/spec/utterson_spec.rb +164 -0
metadata +147 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef4a4e14d3f5b03451d25de3bbd09e7a977ec2b4
+  data.tar.gz: 5fb7fe1675f420b24267cc78e937bf98db2e4d8a
+SHA512:
+  metadata.gz: 3244214395c560032c66ca75d82fe73c319bf1ac527783bb263a9e1676f3910057fc83ae529ba4db99f4ccbc7691ed367d2c5fea9ec514b237f079ad965a04e5
+  data.tar.gz: cc7d65f5cf0689d10ed99a65d62bcc5a1ee82589f2eebd2aefba9a533f5cfc1861ba49b33bd8e6fb0107191fc035dbad07e274b01f9fa6acb82c4861a786ce14

data/README.md ADDED

@@ -0,0 +1,8 @@
+Simple utility to traverse directory of html files and check links in them.
+## Why the name?
+I developed this to help me checking links in my Jekyll powered blog
+and Mr. Utterson is the main character in the [Strange Case of Dr
+Jekyll and Mr
+Hyde](https://en.wikipedia.org/wiki/Strange_Case_of_Dr_Jekyll_and_Mr_Hyde).

data/bin/utterson ADDED

@@ -0,0 +1,12 @@
+#! /bin/env ruby
+$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+require 'trollop'
+require 'utterson'
+opts = Trollop::options do
+  opt :root, "Root directory for the site if it differs from target dir", type: :string
+end
+Utterson.new(opts.merge(dir: ARGV[0])).check

data/lib/utterson.rb ADDED

@@ -0,0 +1,105 @@
+require 'nokogiri'
+require 'net/http'
+require 'timeout'
+class Utterson
+  attr_reader :errors
+  def initialize(opts={})
+    @dir = opts[:dir] || './'
+    @root = opts[:root] || @dir
+    @errors = {}
+    @checked_urls = {}
+    @stats = {errors: 0, files: 0, urls: 0}
+  end
+  def check
+    Dir.glob(File.join(@dir, '**/*.{html,htm}')) do |f|
+      @stats[:files] += 1
+      puts "Checking #{f}"
+      collect_uris_from(f).each do |u|
+        @stats[:urls] += 1
+        check_uri(u, f)
+      end
+    end
+    print_results
+  end
+  def collect_uris_from(f)
+    ret = []
+    doc = Nokogiri::HTML(File.read(f))
+    doc.traverse do |el|
+      ret << el['src'] unless el['src'].nil?
+      ret << el['href'] unless el['href'].nil?
+    end
+    ret
+  end
+  def check_uri(url, file)
+    return if @checked_urls[url]
+    if url =~ /^(https?:)?\/\//
+      check_remote_uri url, file
+    else
+      check_local_uri url, file
+    end
+    @checked_urls[url] = true
+  end
+  def check_remote_uri(url, file)
+    begin
+      uri = URI(url.gsub(/^\/\//, 'http://'))
+    rescue URI::InvalidURIError => e
+      return add_error(file, uri.to_s, e.message)
+    end
+    begin
+      response = Net::HTTP.start(uri.host, uri.port,
+                                 :use_ssl => uri.scheme == 'https') do |http|
+        p = uri.path.empty? ? "/" : uri.path
+        http.head(p)
+      end
+      if response.code =~ /^[^23]/
+        add_error(file, uri.to_s, response)
+      end
+    rescue Timeout::Error
+      add_error(file, uri.to_s, "Reading buffer timed out")
+    rescue Errno::ETIMEDOUT
+      add_error(file, uri.to_s, "Connection timed out")
+    rescue SocketError => e
+      add_error(file, uri.to_s, e.message)
+    end
+  end
+  def check_local_uri(url, file)
+    url.gsub!(/\?.*$/, '')
+    if url =~ /^\//
+      path = File.expand_path(".#{url}", @root)
+    else
+      path = File.expand_path(url, File.dirname(file))
+    end
+    add_error(file, url, "File not found") unless File.exists? path
+  end
+  def add_error(file, url, response)
+    @stats[:errors] += 1
+    @errors[file] = {} if @errors[file].nil?
+    @errors[file][url] = response
+  end
+  def print_results
+    @errors.each do |file, info|
+      puts file
+      info.each do |url, response|
+        s = response.respond_to?(:code) ? "HTTP #{response.code}" : response
+        puts "\t#{url}\n\t\t#{s}"
+      end
+    end
+    if @stats[:errors] == 0
+      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked."
+    else
+      puts "#{@stats[:files]} files with #{@stats[:urls]} urls checked and #{@stats[:errors]} errors found."
+    end
+  end
+end

data/spec/fixtures/dir-structure/1.htm ADDED

File without changes

data/spec/fixtures/dir-structure/2.html ADDED

File without changes

data/spec/fixtures/dir-structure/a/3.htm ADDED

File without changes

data/spec/fixtures/dir-structure/a/b/4.html ADDED

File without changes

data/spec/fixtures/sample.html ADDED

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Title</title>
+    <link type="text/css" rel="stylesheet" href="style.css" />
+    <script type="text/javascript" src="script.js"></script>
+  </head>
+  <body>
+    <h1>I'm a heading!</h1>
+    <p>I'm an <a href="http://example.com">example link</a>.</p>
+    <img src="image.jpg" />
+  </body>
+</html>

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,20 @@
+#require 'rubygems'
+require 'webmock/rspec'
+require 'simplecov'
+SimpleCov.start
+require 'utterson'
+require 'stringio'
+def capture_stdout &block
+  old_stdout = $stdout
+  fake_stdout = StringIO.new
+  $stdout = fake_stdout
+  block.call
+  fake_stdout.string
+ensure
+  $stdout = old_stdout
+end

data/spec/utterson_spec.rb ADDED

@@ -0,0 +1,164 @@
+require 'spec_helper'
+describe Utterson do
+  it "should go through all htm and html files in target dir" do
+    u = Utterson.new(dir: "spec/fixtures/dir-structure")
+    u.stub(:collect_uris_from) {[]}
+    ["spec/fixtures/dir-structure/1.htm",
+     "spec/fixtures/dir-structure/2.html",
+     "spec/fixtures/dir-structure/a/3.htm",
+     "spec/fixtures/dir-structure/a/b/4.html"].each do |file|
+      u.should_receive(:collect_uris_from).with(file)
+    end
+    u.check
+  end
+  it "should check all urls which are found" do
+    u = Utterson.new(dir: "spec/fixtures")
+    u.stub(:check_uri) {}
+    u.should_receive(:check_uri).exactly(4).times
+    u.check
+  end
+  it "should find all uris from sample document" do
+    u = Utterson.new
+    uris = u.collect_uris_from("spec/fixtures/sample.html")
+    uris.should include("script.js")
+    uris.should include("style.css")
+    uris.should include("http://example.com")
+    uris.should include("image.jpg")
+  end
+  describe "#check_uri" do
+    let(:u) {Utterson.new}
+    it "should use remote checking for http protocol" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("http://example.com", "file.html")
+      u.check_uri("http://example.com", "file.html")
+    end
+    it "should use remote checking for https protocol" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("https://example.com", "file.html")
+      u.check_uri("https://example.com", "file.html")
+    end
+    it "should use remote checking when only // is specified" do
+      u.stub(:check_remote_uri) {}
+      u.should_receive(:check_remote_uri).with("//example.com", "file.html")
+      u.check_uri("//example.com", "file.html")
+    end
+    it "should use local checking for relative uris" do
+      u.stub(:check_local_uri) {}
+      u.should_receive(:check_local_uri).with("../file.html", "file.html")
+      u.check_uri("../file.html", "file.html")
+    end
+  end
+  describe "#check_local_uri" do
+    let(:u) {Utterson.new(dir: "spec/fixtures/dir-structure")}
+    it "should not assign error info if file exists" do
+      u.check_local_uri("../sample.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+    it "should assign error info if file doesn't exist" do
+      u.check_local_uri("../sample_not_found.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors["spec/fixtures/dir-structure/1.htm"].should == {"../sample_not_found.html" => "File not found"}
+    end
+    it "should use root directory when urls start with /" do
+      u2 = Utterson.new(dir: "spec/fixtures/dir-structure", root: "spec/fixtures")
+      u2.check_local_uri("/sample.html", "spec/fixtures/dir-structure/1.htm")
+      u2.errors.should be_empty
+    end
+    it "should handle target directory as root for urls starting with / if root is no available" do
+      u.check_local_uri("/2.html", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+    it "should ignore query string when checking local files" do
+      u.check_local_uri("2.html?queryparam=value", "spec/fixtures/dir-structure/1.htm")
+      u.errors.should be_empty
+    end
+  end
+  describe "#check_remote_uri" do
+    let(:u) {Utterson.new}
+    it "should not assign error info if request is successfull" do
+      stub_request(:head, "http://example.com/index.html").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 200, :body => "", :headers => {})
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should be_empty
+    end
+    it "should assign error info if there is error response" do
+      stub_request(:head, "http://example.com/404.html").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 404, :body => "", :headers => {})
+      u.check_remote_uri("http://example.com/404.html", "test.html")
+      puts u.errors.inspect
+      u.errors["test.html"].should_not be_empty
+      u.errors["test.html"]["http://example.com/404.html"].instance_of?(Net::HTTPNotFound).should be_true
+    end
+    it "should add error status from buffer timeouts" do
+      stub_request(:head, "http://example.com/index.html").to_timeout
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "should add error status from connection timeouts" do
+      stub_request(:head, "http://example.com/index.html").to_raise(Errno::ETIMEDOUT)
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "shoud add error status from name resolution errors" do
+      stub_request(:head, "http://example.com/index.html").
+        to_raise(SocketError.new('getaddrinfo: Name or service not known'))
+      u.check_remote_uri("http://example.com/index.html", "test.html")
+      u.errors.should_not be_empty
+    end
+    it "shoud add error status when invalid URI" do
+      URI.stub(:new).and_raise(URI::InvalidURIError)
+      u.check_remote_uri("http://invalid_uri", "test.html")
+      u.errors.should_not be_empty
+    end
+  end
+  describe "#print_results" do
+    it "should output only basic stats if no errors" do
+      u = Utterson.new(dir: "spec/fixtures/dir-structure")
+      output = capture_stdout do
+        u.check
+      end
+      output.should match(/4 files with 0 urls checked/)
+    end
+    it "should output error information" do
+      stub_request(:head, "http://example.com/").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
+        to_return(:status => 404, :body => "", :headers => {})
+      u = Utterson.new(dir: "spec/fixtures")
+      output = capture_stdout do
+        u.check
+      end
+      output.should match("spec/fixtures/sample.html\n\tstyle.css\n\t\tFile not found")
+      output.should match("script.js\n\t\tFile not found")
+      output.should match("image.jpg\n\t\tFile not found")
+      output.should match("http://example.com\n\t\tHTTP 404")
+      output.should match("5 files with 4 urls checked and 4 errors found")
+    end
+  end
+end

metadata ADDED

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: utterson
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Juhamatti Niemelä
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-11-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.15.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.15.2
+description: Traverses all HTML files from given directory and checks links found
+  in them.
+email: iiska@iki.fi
+executables:
+- utterson
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/utterson
+- lib/utterson.rb
+- README.md
+- spec/spec_helper.rb
+- spec/fixtures/dir-structure/a/3.htm
+- spec/fixtures/dir-structure/a/b/4.html
+- spec/fixtures/dir-structure/2.html
+- spec/fixtures/dir-structure/1.htm
+- spec/fixtures/sample.html
+- spec/utterson_spec.rb
+homepage: https://github.com/iiska/utterson
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ~>
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.0.3
+signing_key:
+specification_version: 4
+summary: Friendly HTML crawler and url checker
+test_files:
+- spec/spec_helper.rb
+- spec/fixtures/dir-structure/a/3.htm
+- spec/fixtures/dir-structure/a/b/4.html
+- spec/fixtures/dir-structure/2.html
+- spec/fixtures/dir-structure/1.htm
+- spec/fixtures/sample.html
+- spec/utterson_spec.rb
+has_rdoc: