bad_link_finder 0.1.1 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjQxZjYzNjJjOWExYjY1MDE0Zjk4OGE0YjgwMWQ0YzUxZmIzOWZlMg==
+    M2ZkZmE5YjJiNTMxYThlYmFlYWE2NGM5ZWU5MTExYTA5ZWFiMzJjNw==
   data.tar.gz: !binary |-
-    YjM2ODVlNWE1YmE4ZTllZWUzNDM2NWRhM2I2YjJkNmQyOWYzNjUxZg==
+    YTEwMTBlYWNmMmVlZjY1MTI2YjQ2OWU2ZjVhZTRkZDU1YmE3MWIxYw==
 SHA512:
   metadata.gz: !binary |-
-    N2VhZjg2OWZmZGUzOWQ0YjhmMTEwZWQ1YjE2YjBiOTg5ZTJiMTA0M2ZjM2M3
-    NWY4OTY4ZWMxNGQxMjQ1ZmE3ZTYxNTQyOGJmYjU4OTZkOGI0NmY4NzQzZjhh
-    MDZlYmZiZDdhM2M3M2M0MzA2NWU2NDRlNDBjZTk0YWFiZWJiYzM=
+    M2Y0ZDhkOGRhOTU2ZmIyY2EwOTY4Njk3YTljOTFkMDZjNDZlZDE5YjU3NDI4
+    NmNjMDM1NWQ1NThkY2ZhMzdmNjRhODY3NGY2NGQ1NGI2MzEyNGFmOTM2ZGY1
+    ZGU5NTI4YWUzYWI3NGM1YzRhYWM2MTYxZjhlNWIyOTUzNzlhZGU=
   data.tar.gz: !binary |-
-    ZDIzNjUwYmM0OGRiOTFiNWU0NTQ2MThiMGViOGE5NWJlYWFlMDJjMGI1ZDQ0
-    YjIxZjNlMWY5OWYzMDI1YjEwYWIwYzI1ZWFmMjJkMGUyZWRhOTE4ZGZiMmNh
-    MjA3MmNkNGIwMjE1NTFjNzVkMzQ1NzgzNzNhMTE3ZTZmZjdkMzQ=
+    MWQzMjY3NzMyMDBmMThlMWRmZGZkMGE4YjVmNDRjYTg0ZjA4NDQwNzYyY2Mx
+    ODM5NTFjNjU0MTQ1NzY1ZTM0YTg5OGE3MDAyMTJjYTM0OWU5YmQ5MzZkYTA5
+    ZjNlMzQ0ZDcxNDZjYmE0Y2U1YTU1ZmE2NTAyZGIzYmQ2MThjMzE=

data/lib/bad_link_finder/csv_builder.rb

@@ -2,21 +2,22 @@ require 'csv'
 
 module BadLinkFinder
   class CSVBuilder
-    def initialize(bad_link_map)
-      @bad_link_map = bad_link_map
+    def initialize(csv_output_file)
+      @csv = CSV.new(csv_output_file, encoding: 'UTF-8')
+
+      @csv << ['page_url', 'page_id', 'link', 'error_message', 'raw_error_message']
     end
 
-    def to_s
-      @to_s ||= CSV.generate(encoding: 'UTF-8') do |csv|
-        csv << ['page_url', 'page_id', 'link', 'error_message', 'raw_error_message']
+    def <<(csv_data)
+      link = csv_data[:link]
 
-        @bad_link_map.each do |page_info, bad_links|
-          bad_links.each do |bad_link|
-            exception_message = bad_link.exception.message if bad_link.exception
-            csv << [page_info[:url], page_info[:id], bad_link.link, bad_link.error_message, exception_message]
-          end
-        end
-      end
+      @csv << [
+        csv_data[:url],
+        csv_data[:id],
+        link.link,
+        link.error_message,
+        (link.exception.message if link.exception)
+      ]
     end
   end
 end

data/lib/bad_link_finder/link.rb

@@ -29,7 +29,7 @@ module BadLinkFinder
     rescue Mechanize::RobotsDisallowedError => exception
       record_error("This link is blocked by robots.txt or nofollow attributes", exception)
     rescue Mechanize::Error, Net::HTTP::Persistent::Error, Timeout::Error, Errno::EINVAL,
-           Errno::ECONNRESET, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+           Errno::ECONNRESET, Errno::ETIMEDOUT, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
            Net::ProtocolError, OpenSSL::SSL::SSLError, SocketError => exception # Thanks Net::HTTP
       record_error("The server failed to serve this page properly", exception)
     end

data/lib/bad_link_finder/page_checker.rb

@@ -11,12 +11,19 @@ module BadLinkFinder
 
     attr_reader :page_url
 
-    def bad_links
-      @bad_links ||= @page.links.map do |raw_link|
-        link = @result_cache.fetch(raw_link) || @result_cache.store(raw_link, BadLinkFinder::Link.new(@page_url, raw_link))
+    def each_bad_link(&block)
+      if @bad_links
+        @bad_links.each(&block)
+      else
+        @bad_links = @page.links.map do |raw_link|
+          link = @result_cache.fetch(raw_link) || @result_cache.store(raw_link, BadLinkFinder::Link.new(@page_url, raw_link))
 
-        link unless link.valid?
-      end.compact
+          unless link.valid?
+            yield link
+            next link
+          end
+        end.compact
+      end
     end
   end
 end

data/lib/bad_link_finder/site_checker.rb

@@ -4,31 +4,28 @@ require 'bad_link_finder/page_checker'
 
 module BadLinkFinder
   class SiteChecker
-    def initialize(mirror_dir, host)
+    def initialize(mirror_dir, host, csv_builder)
       @mirror_dir = File.expand_path(mirror_dir)
       @host = host
+      @csv_builder = csv_builder
       @result_cache = BadLinkFinder::ResultCache.new
     end
 
     def run
-      bad_link_map = {}
-      BadLinkFinder::Site.new(@mirror_dir).map do |page|
+      BadLinkFinder::Site.new(@mirror_dir).each do |page|
         page_checker = BadLinkFinder::PageChecker.new(@host, page, @result_cache)
         puts "Checking page #{page.path} as #{page_checker.page_url}"
 
-        bad_links = page_checker.bad_links
-
-        if bad_links.any?
-          page_info = {
+        page_checker.each_bad_link do |link|
+          @csv_builder << {
+            url: page_checker.page_url,
             id: page.id,
-            url: page_checker.page_url
+            link: link
           }
-
-          bad_link_map[page_info] = bad_links
         end
       end
 
-      return bad_link_map
+      nil
     end
   end
 end

data/lib/bad_link_finder/version.rb

@@ -1,3 +1,3 @@
 module BadLinkFinder
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/bad_link_finder.rb

@@ -10,14 +10,17 @@ module BadLinkFinder
 
     raise EnvironmentVariableError.new("MIRROR_DIR '#{ENV['MIRROR_DIR']}' does not exist") unless Dir.exist?(ENV['MIRROR_DIR'])
 
-    bad_link_map = BadLinkFinder::SiteChecker.new(ENV['MIRROR_DIR'], ENV['SITE_HOST']).run
-    csv_builder = CSVBuilder.new(bad_link_map)
-
     report_path = Pathname.new(ENV['REPORT_OUTPUT_FILE'])
     report_path.parent.mkpath
-    report_path.open('w') do |file|
-      file.write(csv_builder)
-    end
+
+    csv_file = report_path.open('w')
+    csv_builder = BadLinkFinder::CSVBuilder.new(csv_file)
+
+    BadLinkFinder::SiteChecker.new(ENV['MIRROR_DIR'], ENV['SITE_HOST'], csv_builder).run
+
+    csv_file.close
+
+    nil
   end
 
   class EnvironmentVariableError < ArgumentError; end

data/test/unit/csv_builder_test.rb

@@ -1,44 +1,101 @@
 require 'test_helper'
 require 'bad_link_finder/csv_builder'
+
+require 'tempfile'
 require 'ostruct'
 require 'csv'
 
 describe BadLinkFinder::CSVBuilder do
 
-  it "flattens out the bad links map into a CSV structure" do
-    bad_link_map = {
-      {url: 'http://www.example.com/example/', id: 'some-article-id'} => [
-        mock_link(link: 'https://www.example.net/external-example.html', error_message: "This link returned a 404", exception: TestException.new('404 not found')),
-        mock_link(link: 'relative-example', error_message: "Nope")
-      ],
-      {url: 'http://www.example.com/example/relative-example'} => [
-        mock_link(
+  before do
+    @report_output_file = Tempfile.new('csv')
+    @report_output_file.unlink
+  end
+
+  after do
+    @report_output_file.close
+  end
+
+  it "writes headers to the output file on creation" do
+    BadLinkFinder::CSVBuilder.new(@report_output_file)
+    @report_output_file.rewind
+    parsed_csv = CSV.parse(@report_output_file.read)
+
+    assert_equal ['page_url', 'page_id', 'link', 'error_message', 'raw_error_message'], parsed_csv.shift
+    assert_empty parsed_csv
+  end
+
+  describe '#<<' do
+    it "writes a link to the CSV" do
+      csv_builder = BadLinkFinder::CSVBuilder.new(@report_output_file)
+
+      csv_builder << {
+        url: 'http://www.example.com/example/',
+        id: 'some-article-id',
+        link: mock_link(link: 'https://www.example.net/external-example.html', error_message: "This link returned a 404", exception: TestException.new('404 not found'))
+      }
+
+      @report_output_file.rewind
+      parsed_csv = CSV.parse(@report_output_file.read)
+
+      parsed_csv.shift # drop headers
+
+      assert_equal [
+        'http://www.example.com/example/',
+        'some-article-id',
+        'https://www.example.net/external-example.html',
+        'This link returned a 404',
+        '404 not found'
+      ], parsed_csv.shift
+    end
+
+    it "ignores missing exceptions" do
+      csv_builder = BadLinkFinder::CSVBuilder.new(@report_output_file)
+
+      csv_builder << {
+        url: 'http://www.example.com/example/',
+        id: 'some-article-id',
+        link: mock_link(link: 'relative-example', error_message: "Nope")
+      }
+
+      @report_output_file.rewind
+      parsed_csv = CSV.parse(@report_output_file.read)
+
+      parsed_csv.shift # drop headers
+
+      assert_equal [
+        'http://www.example.com/example/',
+        'some-article-id',
+        'relative-example',
+        'Nope',
+        nil
+      ], parsed_csv.shift
+    end
+
+    it "ignores missing ids" do
+      csv_builder = BadLinkFinder::CSVBuilder.new(@report_output_file)
+
+      csv_builder << {
+        url: 'http://www.example.com/example/relative-example',
+        link: mock_link(
           link: '/example/?test=true&redirect=http://www.example.com/in-param-url/index.html#section-1',
-          error_message: "What even is this?",
+          error_message: 'What even is this?',
           exception: TestException.new('Test exception')
         )
-      ]
-    }
-
-    csv_builder = BadLinkFinder::CSVBuilder.new(bad_link_map)
-
-    parsed_csv = CSV.parse(csv_builder.to_s)
+      }
 
-    headers = parsed_csv.shift
-    assert_equal ['page_url', 'page_id', 'link', 'error_message', 'raw_error_message'], headers
+      @report_output_file.rewind
+      parsed_csv = CSV.parse(@report_output_file.read)
 
-    assert_equal bad_link_map.values.flatten.count, parsed_csv.count
+      parsed_csv.shift # drop headers
 
-    bad_link_map.each do |page_info, links|
-      links.each do |link|
-        assert parsed_csv.include?([
-          page_info[:url],
-          page_info[:id],
-          link.link,
-          link.error_message,
-          (link.exception.message if link.exception),
-        ])
-      end
+      assert_equal [
+        'http://www.example.com/example/relative-example',
+        nil,
+        '/example/?test=true&redirect=http://www.example.com/in-param-url/index.html#section-1',
+        'What even is this?',
+        'Test exception'
+      ], parsed_csv.shift
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bad_link_finder
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Elliot Crosby-McCullough