bad_link_finder 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2ZkYWM1MjViZmM0M2E3ZDUyZDQ1NmU2MjQ4NDU5Yzk4YjMwZTY3Yg==
+  data.tar.gz: !binary |-
+    MTljZDc2YzUxNmFkZmZjMWNhMzlkYWE1MDg1OWQ2YzU2OWFkYzUzNw==
+SHA512:
+  metadata.gz: !binary |-
+    NjM3MjRkOTVlMGRlYWI0NTliMDViMDI2ZjlkZjI1MGZjNDhjYTMwYWIyNjUy
+    OGJjODRiZmJlYzMzMzI2NzcyZGJhNjE4ZmY4ZjQzZjFlZTMyOWQ4ZDk5MzZm
+    MmRiZWYxYmViYmMwYzJjMjdmNTQyNWU2MTIzMDUzYWE0MmFiYmY=
+  data.tar.gz: !binary |-
+    MTBlYjc4OTU5OWFkOWM5YTE4MDVhNjgzNDI0ZmQxNmQwODcwZWU3NzhkMTlh
+    MzQ0YjhhMzExY2JmZGFmN2M2YmYxOTNhNmRmNDg3M2I3MTQ3NjljMmU2NTg4
+    NmNlZWNkYzIwOTc1ZWRlMjU4ODE2MDI3NDgxODc4NTZkODA4YTc=

data/LICENCE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Elliot Crosby-McCullough
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,13 @@
+# Bad link finder
+
+Crawls a mirrored site and checks that all links either return a successful response or redirect to somewhere that does.
+
+## Usage
+
+Set environment variables:
+
+- `MIRROR_DIR`, to the location of your mirrored site.
+- `REPORT_OUTPUT_FILE`, to the location you'd like the CSV saved to.
+- `SITE_HOST`, to the full host of the live site you'd like to test against, including protocol.  For example, `https://www.example.com`
+
+Then execute `bad_link_finder`.

data/bin/bad_link_finder

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'bad_link_finder'
+
+begin
+  BadLinkFinder.run
+rescue BadLinkFinder::EnvironmentVariableError => e
+  abort "Please check your environment variables: #{e.message}"
+end

data/lib/bad_link_finder/csv_builder.rb

@@ -0,0 +1,22 @@
+require 'csv'
+
+module BadLinkFinder
+  class CSVBuilder
+    def initialize(bad_link_map)
+      @bad_link_map = bad_link_map
+    end
+
+    def to_s
+      @to_s ||= CSV.generate(encoding: 'UTF-8') do |csv|
+        csv << ['page_url', 'link', 'error_message', 'raw_error_message']
+
+        @bad_link_map.each do |page_url, bad_links|
+          bad_links.each do |bad_link|
+            exception_message = bad_link.exception.message if bad_link.exception
+            csv << [page_url, bad_link.link, bad_link.error_message, exception_message]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bad_link_finder/link.rb

@@ -0,0 +1,68 @@
+require 'mechanize'
+
+module BadLinkFinder
+  class Link
+    attr_reader :link, :url, :error_message, :exception
+
+    def initialize(page_url, link)
+      @page_url = page_url
+      @link = link
+      @url = get_url_from_link(link)
+
+      validate_with_request
+
+    rescue URI::InvalidURIError => exception
+      record_error("This link is in a bad format", exception)
+    rescue Mechanize::ResponseCodeError => exception
+      if exception.response_code.to_i == 405 && !@head_unsupported
+        @head_unsupported = true
+        retry
+      else
+        record_error("This request returned a #{exception.response_code}", exception)
+      end
+    rescue Mechanize::UnauthorizedError => exception
+      record_error("This link requires authorisation", exception)
+    rescue Mechanize::UnsupportedSchemeError => exception
+      record_error("This link has a scheme we can't load (should be http or https)", exception)
+    rescue Mechanize::RedirectLimitReachedError => exception
+      record_error("This link might be in a redirect loop", exception)
+    rescue Mechanize::RobotsDisallowedError => exception
+      record_error("This link is blocked by robots.txt or nofollow attributes", exception)
+    rescue Mechanize::Error, Net::HTTP::Persistent::Error, Timeout::Error, Errno::EINVAL,
+           Errno::ECONNRESET, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+           Net::ProtocolError, OpenSSL::SSL::SSLError, SocketError => exception # Thanks Net::HTTP
+      record_error("The server failed to serve this page properly", exception)
+    end
+
+    def valid?
+      @error_message.nil?
+    end
+
+  protected
+
+    def validate_with_request
+      puts "-- testing link #{@link} using #{@url}"
+      sleep 0.1 # Recommended pause for gov.uk rate limiting
+
+      browser = Mechanize.new
+      browser.user_agent = 'GOV.UK link checker'
+
+      if @head_unsupported
+        browser.get(@url)
+      else
+        browser.head(@url)
+      end
+    end
+
+    def get_url_from_link(link)
+      URI.join(@page_url, link).to_s
+    end
+
+    def record_error(message, exception = nil)
+      @error_message = message
+      @exception = exception
+
+      puts "---- found broken link #{@url}: #{message}: #{exception.message if exception}"
+    end
+  end
+end

data/lib/bad_link_finder/page.rb

@@ -0,0 +1,31 @@
+require 'nokogiri'
+
+module BadLinkFinder
+  class Page
+    def initialize(mirror_dir, path)
+      @path = strip_html_ending(path)
+
+      file = mirror_dir + path
+      doc = Nokogiri::HTML(file.read)
+      @links = doc.css('a').map do |a|
+        strip_html_ending(a['href']) unless ignore_link?(a['href'])
+      end.compact
+    end
+
+    attr_reader :path, :links
+
+  protected
+
+    def strip_html_ending(href)
+      if href.start_with?('http')
+        href
+      else
+        href.sub(%r{(?<!\?)(?:index\.html|\.html)(.*)}, '\1')
+      end
+    end
+
+    def ignore_link?(href)
+      href.nil? || href.start_with?('#', 'mailto:')
+    end
+  end
+end

data/lib/bad_link_finder/page_checker.rb

@@ -0,0 +1,22 @@
+require 'bad_link_finder/link'
+
+module BadLinkFinder
+  class PageChecker
+    def initialize(host, page, result_cache)
+      host = host.chomp('/') + '/'
+      @page = page
+      @page_url = URI.join(host, page.path).to_s
+      @result_cache = result_cache
+    end
+
+    attr_reader :page_url
+
+    def bad_links
+      @bad_links ||= @page.links.map do |raw_link|
+        link = @result_cache.fetch(raw_link) || @result_cache.store(raw_link, BadLinkFinder::Link.new(@page_url, raw_link))
+
+        link unless link.valid?
+      end.compact
+    end
+  end
+end

data/lib/bad_link_finder/result_cache.rb

@@ -0,0 +1,24 @@
+# If/when the bad link finder is converted to a set of parallel processes
+# this cache will need to be backed by something threadsafe.
+
+module BadLinkFinder
+  class ResultCache
+    def initialize
+      @cache = {}
+    end
+
+    def store(key, link)
+      @cache[stripped_key(key)] = link
+    end
+
+    def fetch(key)
+      @cache[stripped_key(key)]
+    end
+
+  protected
+
+    def stripped_key(key)
+      key.sub(/#.*$/, '')
+    end
+  end
+end

data/lib/bad_link_finder/site.rb

@@ -0,0 +1,21 @@
+require 'pathname'
+require 'bad_link_finder/page'
+
+module BadLinkFinder
+  class Site
+    include Enumerable
+
+    def initialize(mirror_dir)
+      @mirror_dir = mirror_dir.is_a?(String) ? Pathname.new(mirror_dir) : mirror_dir
+    end
+
+    def each
+      Dir.chdir(@mirror_dir) do
+        Dir.glob('**/*').each do |path|
+          next if File.directory?(path)
+          yield BadLinkFinder::Page.new(@mirror_dir, path)
+        end
+      end
+    end
+  end
+end

data/lib/bad_link_finder/site_checker.rb

@@ -0,0 +1,26 @@
+require 'bad_link_finder/site'
+require 'bad_link_finder/result_cache'
+require 'bad_link_finder/page_checker'
+
+module BadLinkFinder
+  class SiteChecker
+    def initialize(mirror_dir, host)
+      @mirror_dir = File.expand_path(mirror_dir)
+      @host = host
+      @result_cache = BadLinkFinder::ResultCache.new
+    end
+
+    def run
+      bad_link_map = {}
+      BadLinkFinder::Site.new(@mirror_dir).map do |page|
+        page_checker = BadLinkFinder::PageChecker.new(@host, page, @result_cache)
+        puts "Checking page #{page.path} as #{page_checker.page_url}"
+
+        bad_links = page_checker.bad_links
+        bad_link_map[page_checker.page_url] = bad_links if bad_links.any?
+      end
+
+      return bad_link_map
+    end
+  end
+end

data/lib/bad_link_finder/version.rb

@@ -0,0 +1,3 @@
+module BadLinkFinder
+  VERSION = "0.0.1"
+end

data/lib/bad_link_finder.rb

@@ -0,0 +1,24 @@
+require 'bad_link_finder/site_checker'
+require 'bad_link_finder/csv_builder'
+require 'pathname'
+
+module BadLinkFinder
+  def self.run
+    ['MIRROR_DIR', 'REPORT_OUTPUT_FILE', 'SITE_HOST'].each do |var|
+      raise EnvironmentVariableError.new("Missing environment variable #{var}") unless ENV.has_key?(var)
+    end
+
+    raise EnvironmentVariableError.new("MIRROR_DIR '#{ENV['MIRROR_DIR']}' does not exist") unless Dir.exist?(ENV['MIRROR_DIR'])
+
+    bad_link_map = BadLinkFinder::SiteChecker.new(ENV['MIRROR_DIR'], ENV['SITE_HOST']).run
+    csv_builder = CSVBuilder.new(bad_link_map)
+
+    report_path = Pathname.new(ENV['REPORT_OUTPUT_FILE'])
+    report_path.parent.mkpath
+    report_path.open('w') do |file|
+      file.write(csv_builder)
+    end
+  end
+
+  class EnvironmentVariableError < ArgumentError; end
+end

data/test/fixtures/www.example.com/example/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head><title>Example site</title></head>
+  <body>
+    <a href='relative-example.html'>Relative example</a>
+    <a href='relative-example.html'>Relative example</a>
+    <a href='https://www.example.net/external-example.html'>External example</a>
+  </body>
+</html>

data/test/fixtures/www.example.com/example/relative-example.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head><title>Example site</title></head>
+  <body>
+    <!-- Included -->
+    <a href='/example/index.html?test=true&redirect=http://www.example.com/in-param-url/index.html#section-1'></a>
+    <a href=''></a>
+
+    <!-- Excluded -->
+    <a></a>
+    <a href='#section-2'></a>
+    <a href='mailto:test@example.com'></a>
+  </body>
+</html>

data/test/fixtures/www.example.com/index.html

@@ -0,0 +1,7 @@
+<!DOCTYPE html>
+<html>
+  <head><title>Example site</title></head>
+  <body>
+    <a href='/example/index.html'>Example 1</a>
+  </body>
+</html>

data/test/integration/bad_link_finder_test.rb

@@ -0,0 +1,44 @@
+require 'test_helper'
+require 'webmock/minitest'
+require 'bad_link_finder'
+
+describe BadLinkFinder do
+
+  before do
+    stub_request(:any, 'http://www.example.com/example/').to_return(status: 200)
+    stub_request(:any, 'http://www.example.com/example/relative-example').to_return(status: 302, headers: {'Location' => 'http://www.example.com/example/'})
+    stub_request(:any, 'https://www.example.net/external-example.html').to_return(status: 500)
+    stub_request(:any, 'http://www.example.com/example/?test=true&redirect=http://www.example.com/in-param-url/index.html').to_return(status: 404)
+
+    ENV['MIRROR_DIR'] = (FIXTURES_ROOT+'www.example.com').to_s
+    ENV['REPORT_OUTPUT_FILE'] = (TMP_ROOT+'bad_links.csv').to_s
+    ENV['SITE_HOST'] = 'http://www.example.com/'
+  end
+
+  it "finds all broken links and exports to a CSV" do
+    BadLinkFinder.run
+
+    csv_string = File.read(ENV['REPORT_OUTPUT_FILE'])
+
+    assert_match 'http://www.example.com/example/', csv_string
+  end
+
+  it "complains if key variables are missing" do
+    ['MIRROR_DIR', 'REPORT_OUTPUT_FILE', 'SITE_HOST'].each do |var|
+      ENV.delete(var)
+
+      assert_raises(BadLinkFinder::EnvironmentVariableError) do
+        BadLinkFinder.run
+      end
+    end
+  end
+
+  it "complains if the MIRROR_DIR does not exist" do
+    ENV['MIRROR_DIR'] = (FIXTURES_ROOT+'this_does_not_exist').to_s
+
+    assert_raises(BadLinkFinder::EnvironmentVariableError) do
+      BadLinkFinder.run
+    end
+  end
+
+end

data/test/support/matchers.rb

@@ -0,0 +1,5 @@
+class Minitest::Test
+  def assert_same_elements(array1, array2)
+    assert_equal array1.to_set, array2.to_set, "Different elements in #{array1.inspect} and #{array2.inspect}"
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+require 'bundler/setup'
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift File.expand_path("..", __FILE__)
+
+require 'minitest/autorun'
+require 'support/matchers'
+
+require 'pathname'
+APP_ROOT = Pathname.new(File.join(File.dirname(__FILE__), '..'))
+FIXTURES_ROOT = APP_ROOT+'test/fixtures'
+TMP_ROOT = APP_ROOT+'tmp'

data/test/unit/csv_builder_test.rb

@@ -0,0 +1,45 @@
+require 'test_helper'
+require 'bad_link_finder/csv_builder'
+require 'ostruct'
+require 'csv'
+
+describe BadLinkFinder::CSVBuilder do
+
+  it "flattens out the bad links map into a CSV structure" do
+    bad_link_map = {
+      'http://www.example.com/example/' => [
+        mock_link(link: 'https://www.example.net/external-example.html', error_message: "This link returned a 404", exception: TestException.new('404 not found')),
+        mock_link(link: 'relative-example', error_message: "Nope")
+      ],
+      'http://www.example.com/example/relative-example' => [
+        mock_link(
+          link: '/example/?test=true&redirect=http://www.example.com/in-param-url/index.html#section-1',
+          error_message: "What even is this?",
+          exception: TestException.new('Test exception')
+        )
+      ]
+    }
+
+    csv_builder = BadLinkFinder::CSVBuilder.new(bad_link_map)
+
+    parsed_csv = CSV.parse(csv_builder.to_s)
+
+    headers = parsed_csv.shift
+    assert_equal ['page_url', 'link', 'error_message', 'raw_error_message'], headers
+
+    assert_equal bad_link_map.values.flatten.count, parsed_csv.count
+
+    bad_link_map.each do |page_url, links|
+      links.each do |link|
+        assert parsed_csv.include?([page_url, link.link, link.error_message, (link.exception.message if link.exception)])
+      end
+    end
+  end
+
+  def mock_link(attrs)
+    OpenStruct.new(attrs)
+  end
+
+  class TestException < Exception; end
+
+end

data/test/unit/link_test.rb

@@ -0,0 +1,70 @@
+require 'test_helper'
+require 'webmock/minitest'
+require 'bad_link_finder/link'
+
+describe BadLinkFinder::Link do
+
+  describe '#valid?' do
+    it "approves fully qualified urls which get a good response" do
+      stub_url("http://www.example.com", 200)
+      link = build_link('http://www.example.com')
+
+      assert link.valid?
+    end
+
+    it "approves relative paths which get a good response" do
+      stub_url("http://www.example.com/somewhere/an-example-path", 200)
+      link = build_link('an-example-path', page_url: 'http://www.example.com/somewhere/')
+
+      assert link.valid?
+    end
+
+    it "approves absolute paths which get a good response" do
+      stub_url("http://www.example.com/an-example-path", 200)
+      link = build_link('/an-example-path', page_url: 'http://www.example.com/somewhere/')
+
+      assert link.valid?
+    end
+
+    it "reports malformed links without checking the internet" do
+      link = build_link('htt[]://{an-example-path}')
+
+      refute link.valid?
+      assert_equal "This link is in a bad format", link.error_message
+    end
+
+    it "reports links returning failure status codes" do
+      stub_url("http://www.example.com/an-example-path", 404)
+      link = build_link('/an-example-path')
+
+      refute link.valid?
+      assert_equal "This request returned a 404", link.error_message
+    end
+
+    it "reports URLs returning failure status codes" do
+      stub_url("https://www.example.net/an-external-failure", 500)
+      link = build_link('https://www.example.net/an-external-failure')
+
+      refute link.valid?
+      assert_equal "This request returned a 500", link.error_message
+    end
+
+    it "retries 405s as GET requests" do
+      stub_request(:head, "http://www.example.com/an-example-path").to_return(status: 405)
+      stub_request(:get, "http://www.example.com/an-example-path").to_return(status: 200)
+      link = build_link('/an-example-path')
+
+      assert link.valid?
+    end
+  end
+
+  def stub_url(url, status)
+    stub_request(:any, url).to_return(status: status)
+  end
+
+  def build_link(link_path, opts = {})
+    page_url = opts[:page_url] || 'http://www.example.com'
+    BadLinkFinder::Link.new(page_url, link_path)
+  end
+
+end

data/test/unit/page_checker_test.rb

@@ -0,0 +1,22 @@
+require 'test_helper'
+require 'bad_link_finder/page_checker'
+require 'bad_link_finder/page'
+require 'bad_link_finder/result_cache'
+
+describe BadLinkFinder::PageChecker do
+
+  describe "#page_url" do
+    it "correctly merges the host with the page path" do
+      assert_equal 'http://www.example.com/', build_page_checker('index.html').page_url.to_s
+      assert_equal 'http://www.example.com/example/', build_page_checker('example/index.html').page_url.to_s
+      assert_equal 'http://www.example.com/example/relative-example', build_page_checker('example/relative-example.html').page_url.to_s
+    end
+  end
+
+  def build_page_checker(path)
+    site_mirror = FIXTURES_ROOT+'www.example.com'
+    page = BadLinkFinder::Page.new(site_mirror, path)
+    BadLinkFinder::PageChecker.new('http://www.example.com/', page, BadLinkFinder::ResultCache.new)
+  end
+
+end

data/test/unit/page_test.rb

@@ -0,0 +1,50 @@
+require 'test_helper'
+require 'bad_link_finder/page'
+
+describe BadLinkFinder::Page do
+
+  it "strips index.html and .html from the page path" do
+    assert_equal '', build_page('index.html').path.to_s
+    assert_equal 'example/', build_page('example/index.html').path.to_s
+    assert_equal 'example/relative-example', build_page('example/relative-example.html').path.to_s
+  end
+
+  it "finds absolute paths, stripping index.html and .html" do
+    assert_equal ['/example/'], build_page('index.html').links.map(&:to_s)
+  end
+
+  it "finds relative paths, stripping index.html and .html" do
+    assert build_page('example/index.html').links.map(&:to_s).include?('relative-example')
+  end
+
+  it "finds and preserves external URLs" do
+    assert build_page('example/index.html').links.map(&:to_s).include?('https://www.example.net/external-example.html')
+  end
+
+  it "preserves params and anchors on internal links" do
+    page = build_page('example/relative-example.html')
+    assert page.links.map(&:to_s).include?('/example/?test=true&redirect=http://www.example.com/in-param-url/index.html#section-1')
+  end
+
+  it "includes links with empty href" do
+    assert build_page('example/relative-example.html').links.map(&:to_s).include?('')
+  end
+
+  it "excludes links with no href" do
+    refute build_page('example/relative-example.html').links.include?(nil)
+  end
+
+  it "excludes links with an href containing only an anchor reference" do
+    refute build_page('example/relative-example.html').links.map(&:to_s).include?('#section-2')
+  end
+
+  it "excludes mailto links" do
+    refute build_page('example/relative-example.html').links.map(&:to_s).include?('mailto:test@example.com')
+  end
+
+  def build_page(path)
+    site_mirror = FIXTURES_ROOT+'www.example.com'
+    BadLinkFinder::Page.new(site_mirror, path)
+  end
+
+end

data/test/unit/result_cache_test.rb

@@ -0,0 +1,38 @@
+require 'test_helper'
+require 'bad_link_finder/result_cache'
+
+describe BadLinkFinder::ResultCache do
+
+  before do
+    @cache = BadLinkFinder::ResultCache.new
+  end
+
+  it "returns a cache hit for URLs which differ only by anchor" do
+    @cache.store('http://www.example.com#test123', 'value')
+    assert_equal 'value', @cache.fetch('http://www.example.com#test567')
+
+    @cache.store('http://www.example.com?test=true#test123', 'value')
+    assert_equal 'value', @cache.fetch('http://www.example.com?test=true#test567')
+
+    @cache.store('http://www.example.com?test=true#test123', 'value')
+    refute_equal 'value', @cache.fetch('http://www.example.com?test=false#test567')
+  end
+
+  describe "#store" do
+    it "returns the item stored" do
+      assert_equal 'value', @cache.store('key', 'value')
+    end
+  end
+
+  describe "#fetch" do
+    it "returns fetched items on a hit" do
+      @cache.store('key', 'value')
+      assert_equal 'value', @cache.fetch('key')
+    end
+
+    it "returns nil on a miss" do
+      assert_nil @cache.fetch('missing-key')
+    end
+  end
+
+end

data/test/unit/site_test.rb

@@ -0,0 +1,22 @@
+require 'test_helper'
+require 'bad_link_finder/site'
+
+describe BadLinkFinder::Site do
+
+  before do
+    @site_mirror = FIXTURES_ROOT+'www.example.com'
+  end
+
+  describe '#each' do
+    it "loads all files from a directory and passes on the host" do
+      site_map = [
+        '',
+        'example/',
+        'example/relative-example'
+      ]
+
+      assert_same_elements site_map, BadLinkFinder::Site.new(@site_mirror).map { |page| page.path.to_s }
+    end
+  end
+
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: bad_link_finder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Elliot Crosby-McCullough
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: gem_publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+description: Crawls a static site mirror testing all links.  Reports links which don't
+  return 200 or redirect to a 200.
+email:
+- elliot.cm@gmail.com
+executables:
+- bad_link_finder
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/bad_link_finder
+- lib/bad_link_finder/csv_builder.rb
+- lib/bad_link_finder/link.rb
+- lib/bad_link_finder/page.rb
+- lib/bad_link_finder/page_checker.rb
+- lib/bad_link_finder/result_cache.rb
+- lib/bad_link_finder/site.rb
+- lib/bad_link_finder/site_checker.rb
+- lib/bad_link_finder/version.rb
+- lib/bad_link_finder.rb
+- README.md
+- LICENCE.txt
+- test/fixtures/www.example.com/example/index.html
+- test/fixtures/www.example.com/example/relative-example.html
+- test/fixtures/www.example.com/index.html
+- test/integration/bad_link_finder_test.rb
+- test/support/matchers.rb
+- test/test_helper.rb
+- test/unit/csv_builder_test.rb
+- test/unit/link_test.rb
+- test/unit/page_checker_test.rb
+- test/unit/page_test.rb
+- test/unit/result_cache_test.rb
+- test/unit/site_test.rb
+homepage: http://github.com/alphagov/bad_link_finder
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 2.1.11
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Tests links in static site mirrors
+test_files:
+- test/fixtures/www.example.com/example/index.html
+- test/fixtures/www.example.com/example/relative-example.html
+- test/fixtures/www.example.com/index.html
+- test/integration/bad_link_finder_test.rb
+- test/support/matchers.rb
+- test/test_helper.rb
+- test/unit/csv_builder_test.rb
+- test/unit/link_test.rb
+- test/unit/page_checker_test.rb
+- test/unit/page_test.rb
+- test/unit/result_cache_test.rb
+- test/unit/site_test.rb