ndd-url_checker 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ffa576d85139cd2300b55ba9c340fdda8608850
-  data.tar.gz: 58835f36306bd29f9375f0f6c65afc7045b7e411
+  metadata.gz: 99d36713ad66fe4e91a94f8244232de75f4c9e2c
+  data.tar.gz: 49b3313cc7ef54b102f1af8a2e921f9b96e5c70e
 SHA512:
-  metadata.gz: 84a715175f934072d1b5ec7244302d4c8385d7db3d724d1ac7596e9e98ca00f58bccf12f68c4d88ef4cac4fb865f7ea192f435bcc939783e0dd8a4fcec77281b
-  data.tar.gz: 62ee300dbe403c80a42e8387dc79657549c9bbf0f0be3d3788ed692d2847517bc70ecd460a130be42517d6086b909885b7e47dc29c48c79f72e5ab0eebb80e37
+  metadata.gz: eb756c557e6c76cb1f5ba947b383e3b3a5baec440f7b4ae1763251744fb925050a5724588d779ea51f1901948abf2777c77bd5f86c57734117a533794144a7b6
+  data.tar.gz: 3d1aa11d6dec0ea8159bb643b6cebe52f33d84b979e0f2bd858356085a616e5c170af888e72cb8f5872a0617421c0e34541b5ace207e12eaf7ff537a6d3e1727

data/CHANGELOG.md

@@ -1,5 +1,12 @@
-# NDD URL Checker changelog
+# NDD URL Checker changelog
+
+## Version 0.2.0
+
+- Switched to named parameters
+- Fixed documentation
+- Removed the `UrlChecker#validate` method.
+- Changed the type of the return value of `UrlChecker#check`.
 
 ## Version 0.1.0
 
-* initial commit
+- initial commit

data/Guardfile

@@ -1,4 +1,3 @@
-
 # ----------------------------------------------------------------------------------------------------------------------
 # bundler
 # ----------------------------------------------------------------------------------------------------------------------
@@ -31,6 +30,7 @@ guard :rspec, cmd: 'bundle exec rspec' do
 
   # ----- lib directory
   watch(%r{^lib/(.+)\.rb$}) { |m| %W(spec/#{m[1]}_spec.rb) }
+  watch(%r{^lib/(.+)\.(.+)\.erb$}) { |m| %W(spec/#{m[1]}_spec.rb) }
 
   # ----- spec directory
   watch(%r{^spec/.+_spec\.rb$})

data/README.md

@@ -5,13 +5,73 @@
 [![Code Climate](https://codeclimate.com/github/ddidier/ndd-url_checker/badges/gpa.svg)](https://codeclimate.com/github/ddidier/ndd-url_checker)
 [![Test Coverage](https://codeclimate.com/github/ddidier/ndd-url_checker/badges/coverage.svg)](https://codeclimate.com/github/ddidier/ndd-url_checker)
 
-URL validator.
+An URL validator.
+
+The API documentation can be find at [RubyDoc](http://www.rubydoc.info/github/ddidier/ndd-url_checker).
 
 ## Prerequisites
 
-This gem is tested with Ruby 2.1.x.
+This gem requires Ruby 2.x and is tested with:
+
+- Ruby 2.0.0
+- Ruby 2.1.x
+
+## Usage
+
+This gem provides several types of URL checker which may be composed using the [decorator pattern](http://en.wikipedia.org/wiki/Decorator_pattern). An URL checker exposes a [`#check(*urls)`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/AbstractUrlChecker#check-instance_method) method which has 2 variants:
+
+- if a single URL is passed as an argument, it returns a single [`NDD::UrlChecker::Status`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/Status)
+- if multiple URL are passed as arguments, it returns an array of [`NDD::UrlChecker::Status`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/Status)
+
+A status has a code reflecting the result of the URL check. For the time being:
+
+- valid codes are `direct` and `redirected`
+- invalid codes are `failed`, `too_many_redirects` and `unknown_host`
+
+### BlockingUrlChecker
+
+[`NDD::UrlChecker::BlockingUrlChecker`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/BlockingUrlChecker) provides a serial URL checker using the standard [`Net:HTTP`](http://ruby-doc.org/stdlib-2.1.5/libdoc/net/http/rdoc/Net/HTTP.html) implementation.
+
+```ruby
+checker = NDD::UrlChecker::BlockingUrlChecker.new
+
+status = checker.check('http://www.invalid123456789.com/')
+status.valid? # => false
+status.code   # => :unknown_host
+
+status = checker.check('http://www.google.com/')
+status.valid? # => true
+status.code   # => :direct
+
+statuses = checker.check('http://www.invalid123456789.com/', 'http://www.google.com/')
+statuses[0].uri    # => 'http://www.invalid123456789.com/'
+statuses[0].valid? # => false
+statuses[0].code   # => :unknown_host
+statuses[1].uri    # => 'http://www.google.com/'
+statuses[1].valid? # => true
+statuses[1].code   # => :direct
+```
+
+### ParallelUrlChecker
+
+But this will be very time consuming if there is a lot of URL to check. Meet [`NDD::UrlChecker::ParallelUrlChecker`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/ParallelUrlChecker) which enables a very significant processing boost. For the time being, only a forked implementation is provided but a threaded one is planed.
+
+```ruby
+checker = NDD::UrlChecker::ParallelUrlChecker.new(parallelism: 100)
+checker.check('http://www.invalid123456789.com/')
+checker.check('http://www.google.com/')
+```
+
+### ReportingUrlChecker
+
+For a nice looking report, use [`NDD::UrlChecker::ReportingUrlChecker`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/ReportingUrlChecker) which enables reporting capabilities using ERB templates. Several built-in templates are provided: CSV (planed), HTML and JSON (planed).
 
-## TODO
+```ruby
+checker = NDD::UrlChecker:: ReportingUrlChecker.new(delegate_checker)
+statuses = checker.check('http://www.invalid123456789.com/', 'http://www.google.com/')
+report_as_text = checker.report(:html, '/some/report.html')
+report_as_text = checker.report('/some/template.erb', '/some/report.html')
+```
 
 ## Copyright
 

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/ndd/url_checker/abstract_url_checker.rb

@@ -4,27 +4,18 @@ module NDD
   module UrlChecker
 
     # Abstract class, not very ruby-ish :)
+    # @abstract
     # @author David DIDIER
     class AbstractUrlChecker
 
-      # Checks that the given URL are valid.
-      # If there is only a single URL parameter, returns a NDD::UrlChecker::Status.
-      # If there is only multiple URL parameters, returns a Hash of NDD::UrlChecker::Status indexed by their URI.
-      # @param urls [String|Array<String>] the URLs to check.
-      # @return [NDD::UrlChecker::Status|Hash<String => NDD::UrlChecker::Status>]
+      # Checks that the given URLs are valid.
+      # @param urls [String, Array<String>] the URLs to check
+      # @return [NDD::UrlChecker::Status, Array<NDD::UrlChecker::Status>] a single status for a single URL, an array
+      #         of status for multiple parameters
       def check(*urls)
         raise 'NDD::UrlChecker::UrlChecker#check must be implemented'
       end
 
-      # Validates that the given URL are valid.
-      # If there is only a single URL parameter, returns a boolean.
-      # If there is only multiple URL parameters, returns a Hash of boolean indexed by their URI.
-      # @param urls [String|Array<String>] the URLs to validate.
-      # @return [NDD::UrlChecker::Status|Hash<String => Boolean>]
-      def validate(*urls)
-        raise 'NDD::UrlChecker::UrlChecker#validate must be implemented'
-      end
-
     end
   end
 end

data/lib/ndd/url_checker/blocking_url_checker.rb

@@ -7,47 +7,36 @@ require 'net/https'
 module NDD
   module UrlChecker
 
-    # An URL checker using the blocking Net::HTTP class.
+    # An URL checker using the blocking {Net::HTTP} class.
     # @author David DIDIER
     class BlockingUrlChecker < AbstractUrlChecker
 
       # Create a new instance.
-      # @param [Fixnum] maximum_redirects the maximum number of redirects before failing.
-      # @param [Fixnum] timeout the number of seconds to wait before failing.
-      def initialize(maximum_redirects=5, timeout=5)
+      # @param maximum_redirects [Fixnum] the maximum number of redirects to follow before failing.
+      # @param timeout [Fixnum] the number of seconds to wait before failing.
+      def initialize(maximum_redirects: 5, timeout: 5)
         @logger = Logging.logger[self]
         @maximum_redirects = maximum_redirects
         @timeout = timeout
       end
 
-      # Checks that the given URL are valid.
-      # If there is only a single URL parameter, returns a NDD::UrlChecker::Status.
-      # If there is only multiple URL parameters, returns a Hash of NDD::UrlChecker::Status indexed by their URI.
-      # @param [String|Array<String>] urls
-      # @return [NDD::UrlChecker::Status|Hash<String => NDD::UrlChecker::Status>]
+      # Checks that the given URLs are valid.
+      # @param urls [String, Array<String>] the URLs to check
+      # @return [NDD::UrlChecker::Status, Array<NDD::UrlChecker::Status>] a single status for a single URL, an array
+      #         of status for multiple parameters
       def check(*urls)
         @logger.info "Checking #{urls.size} URL(s)"
-        return check_single(urls.first) if urls.size == 1
-        Hash[urls.map { |url| [url, check_single(url)] }]
-      end
-
-      # Validates that the given URL are valid.
-      # If there is only a single URL parameter, returns a boolean.
-      # If there is only multiple URL parameters, returns a Hash of boolean indexed by their URI.
-      # @param [String|Array<String>] urls
-      # @return [NDD::UrlChecker::Status|Hash<String => Boolean>]
-      def validate(*urls)
-        @logger.info "Validating #{urls.size} URL(s)"
-        return validate_single(urls.first) if urls.size == 1
-        Hash[urls.map { |url| [url, validate_single(url)] }]
+        return check_single(urls.first) if urls.size < 2
+        urls.map { |url| check_single(url) }
       end
 
 
+      # -------------------------------------------------------------------------------------------------- private -----
       private
 
       # Checks that the given URL is valid.
-      # @param [String] url
-      # @return [NDD::UrlChecker::Status]
+      # @param url [String] the URL to check
+      # @return [NDD::UrlChecker::Status]
       def check_single(url)
         begin
           @logger.debug "Checking: #{url}"
@@ -63,18 +52,10 @@ module NDD
         status
       end
 
-      # Validates that the given URL are valid.
-      # @param [String] url
-      # @return [Boolean]
-      def validate_single(url)
-        @logger.debug "Validating: #{url}"
-        check_single(url).valid?
-      end
-
-      # Checks that the given URL is valid.
-      # @param [URI::HTTP] uri the URI to check
-      # @param [NDD::UrlChecker::Status] status the current status of the stack
-      # @return [NDD::UrlChecker::Status]
+      # Checks that the given URI is valid.
+      # @param uri [URI::HTTP] the URI to check.
+      # @param status [NDD::UrlChecker::Status] the current status of the stack.
+      # @return [NDD::UrlChecker::Status]
       def check_uri(uri, status)
         if status.uris.size() > @maximum_redirects
           return status.too_many_redirects
@@ -122,7 +103,7 @@ module NDD
         end
       end
 
-      # FIXME: platform dependent?
+      # FIXME: platform dependent?
       UNKNOWN_HOST_MESSAGE = 'getaddrinfo: Name or service not known'
 
       def unknown_host?(error)

data/lib/ndd/url_checker/forked_url_checker.rb

@@ -9,49 +9,43 @@ module NDD
 
     # An URL checker using forks to parallelize processing. To be used with MRI.
     # @author David DIDIER
+    # @attr_reader delegate [#check] the delegate URL checker.
+    # @attr_reader parallelism [Fixnum] the number of processes.
     class ForkedUrlChecker < AbstractUrlChecker
 
       attr_reader :delegate
       attr_reader :parallelism
 
       # Create a new instance.
-      # @param [AbstractUrlChecker] delegate_checker defaults to {NDD::UrlChecker::BlockingUrlChecker}.
-      # @param [Fixnum] parallelism the number of processes.
-      def initialize(delegate_checker=nil, parallelism=10)
+      # @param delegate_checker [AbstractUrlChecker] defaults to {NDD::UrlChecker::BlockingUrlChecker}.
+      # @param parallelism [Fixnum] the number of processes.
+      def initialize(delegate_checker: nil, parallelism: 10)
         @logger = Logging.logger[self]
         @delegate = delegate_checker || BlockingUrlChecker.new
         @parallelism = parallelism
       end
 
+      # Checks that the given URLs are valid.
+      # @param urls [String, Array<String>] the URLs to check
+      # @return [NDD::UrlChecker::Status, Array<NDD::UrlChecker::Status>] a single status for a single URL, an array
+      #         of status for multiple parameters
       def check(*urls)
         return delegate.check(*urls) if urls.size < 2
-        process(urls, :check)
-      end
-
-      def validate(*urls)
-        return delegate.validate(*urls) if urls.size < 2
-        process(urls, :validate)
-      end
-
 
-      private
-
-      def process(urls, method)
         # for receiving results
         result_pipe = Cod.pipe
         # partition the URLs, but not too much :)
         url_slices = partition(urls, [parallelism, urls.size].min)
         # and distribute them among the workers
         pids = url_slices.each_with_index.map do |url_slice, index|
-          fork { Worker.new(index, result_pipe, delegate).send(method, url_slice) }
+          fork { Worker.new(index, result_pipe, delegate).check(url_slice) }
         end
 
         # read back the results
-        results = urls.reduce({}) do |hash, _|
+        results = urls.map do |_|
           result = result_pipe.get
-          hash.merge!(result)
           @logger.debug("Processed URLs #{hash.size}/#{urls.size}")
-          hash
+          result
         end
 
         # kill all the workers
@@ -60,9 +54,15 @@ module NDD
         results
       end
 
-      # Evenly distributes data into buckets. For example:
-      #     partition([1, 2, 3], 2) => [[1, 3], [2]]
-      #     partition([1, 2, 3, 4, 5, 6], 3) => [[1, 4], [2, 5], [3, 6]]
+
+      # -------------------------------------------------------------------------------------------------- private -----
+      private
+
+      # Evenly distributes data into buckets.
+      #
+      # @example
+      #     partition([1, 2, 3], 2)           #=> [[1, 3], [2]]
+      #     partition([1, 2, 3, 4, 5, 6], 3)  #=> [[1, 4], [2, 5], [3, 6]]
       def partition(data, buckets)
         Array.new.tap do |slices|
           buckets.times.each { |_| slices << Array.new }
@@ -71,7 +71,9 @@ module NDD
       end
 
 
+      # ---------------------------------------------------------------------------------------------------- class -----
       # A simple worker class processing URL one by one.
+      # @private
       class Worker
         def initialize(id, result_pipe, url_checker)
           @logger = Logging.logger[self]
@@ -83,18 +85,12 @@ module NDD
         def check(urls)
           @logger.debug("[worker #{@id}] Checking #{urls.size} URLs")
           urls.each do |url|
-            @result_pipe.put({url => @url_checker.check(url)})
-          end
-        end
-
-        def validate(urls)
-          @logger.debug("[worker #{@id}] Validating #{urls.size} URLs")
-          urls.each do |url|
-            @result_pipe.put({url => @url_checker.validate(url)})
+            @result_pipe.put(@url_checker.check(url))
           end
         end
       end
 
+      private_constant :Worker
     end
   end
 end

data/lib/ndd/url_checker/parallel_url_checker.rb

@@ -7,39 +7,40 @@ require 'ndd/url_checker/threaded_url_checker'
 module NDD
   module UrlChecker
 
-    # Wraps an instance of ThreadedUrlChecker or ForkedUrlChecker
-    # depending of the underlying Ruby implementation.
+    # Wraps an instance of {NDD::UrlChecker::ThreadedUrlChecker} or {NDD::UrlChecker::ForkedUrlChecker}
+    # depending of the underlying Ruby implementation.
     # @author David DIDIER
+    # @attr_reader delegate [#check] the delegate URL checker.
     class ParallelUrlChecker < AbstractUrlChecker
 
       attr_reader :delegate
 
       # Create a new instance.
-      # @param [AbstractUrlChecker] delegate_checker defaults to {NDD::UrlChecker::BlockingUrlChecker}.
-      # @param [Fixnum] parallelism the number of threads or processes.
-      def initialize(delegate_checker=nil, parallelism=10)
+      # @param delegate_checker [AbstractUrlChecker] defaults to {NDD::UrlChecker::BlockingUrlChecker}.
+      # @param parallelism [Fixnum] the number of threads or processes.
+      def initialize(delegate_checker: nil, parallelism: 10)
         @logger = Logging.logger[self]
 
         @logger.debug "Ruby engine is #{RUBY_ENGINE}"
         if RUBY_ENGINE == 'jruby'
           @logger.info 'Creating a threaded URL checker'
-          parallel_checker = ThreadedUrlChecker.new(delegate_checker, parallelism)
+          parallel_checker = ThreadedUrlChecker.new(delegate_checker: delegate_checker, parallelism: parallelism)
         else
           @logger.info 'Creating a forked URL checker'
-          parallel_checker = ForkedUrlChecker.new(delegate_checker, parallelism)
+          parallel_checker = ForkedUrlChecker.new(delegate_checker: delegate_checker, parallelism: parallelism)
         end
 
         @delegate = parallel_checker
       end
 
+      # Checks that the given URLs are valid.
+      # @param urls [String, Array<String>] the URLs to check
+      # @return [NDD::UrlChecker::Status, Array<NDD::UrlChecker::Status>] a single status for a single URL, an array
+      #         of status for multiple parameters
       def check(*urls)
         @delegate.check(*urls)
       end
 
-      def validate(*urls)
-        @delegate.validate(*urls)
-      end
-
     end
   end
 end

data/lib/ndd/url_checker/reporting_url_checker.html.erb

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
+  <style>
+    body {
+      padding: 1em;
+    }
+  </style>
+
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js"></script>
+  <script language="JavaScript">
+    $(document).ready(function() {
+      $('input[type=radio][name=urlsFilter]').change(function() {
+        if (this.value == 'all') {
+          $('tr').show();
+        }
+        else if (this.value == 'invalid') {
+          $('tr').not('.danger').hide();
+        }
+      });
+    });
+  </script>
+</head>
+
+<body>
+
+<h1>URL Checker Report</h1>
+
+<table class="table table-condensed table-hover" style="width: auto">
+  <tbody>
+  <tr>
+    <td>User CPU time</td>
+    <td><%= benchmark.utime %> second(s)</td>
+  </tr>
+  <tr>
+    <td>System CPU time</td>
+    <td><%= benchmark.stime %> second(s)</td>
+  </tr>
+  <tr>
+    <td>Total time</td>
+    <td><%= benchmark.total %> second(s)</td>
+  </tr>
+  <tr>
+    <td>Elapsed real time</td>
+    <td><%= benchmark.real %> second(s)</td>
+  </tr>
+  <tr>
+    <td>Number of URLs</td>
+    <td><%= results.size %></td>
+  </tr>
+  <tr>
+    <td>Average time for 1 URL</td>
+    <td><%= benchmark.real/results.size %> second(s)</td>
+  </tr>
+  </tbody>
+</table>
+
+<div class="radio">
+  <label>
+    <input type="radio" name="urlsFilter" id="showAllUrls" value="all" checked="true" />
+    Show all URLs
+  </label>
+</div>
+<div class="radio">
+  <label>
+    <input type="radio" name="urlsFilter" id="showInvalidUrls" value="invalid" />
+    Show invalid URLs
+  </label>
+</div>
+
+<table class="table table-condensed table-hover" style="width: auto">
+  <thead>
+  <tr>
+    <th>#</th>
+    <th>URL</th>
+    <th>Result</th>
+    <th>Comment</th>
+  </tr>
+  </thead>
+  <tbody>
+  <% results.each_with_index do |status, index| %>
+      <tr class="<%= status.invalid? ? 'danger' : '' %>">
+        <td><%= index+1 %></td>
+        <td><a href="<%= status.uri %>" target="_blank"><%= status.uri %></a></td>
+        <td><%= status.class %></td>
+        <td><% if status.invalid? %><%= status.error %><% end %></td>
+      </tr>
+  <% end %>
+  </tbody>
+</table>
+
+</body>
+</html>

data/lib/ndd/url_checker/reporting_url_checker.rb

@@ -0,0 +1,83 @@
+require 'benchmark'
+require 'erb'
+require 'logging'
+require 'ndd/url_checker'
+require 'ostruct'
+
+module NDD
+  module UrlChecker
+
+    # Wraps an instance of URL checker and provides reporting capabilities using ERB templates.
+    # @author David DIDIER
+    # @attr_reader delegate [#check] the delegate URL checker.
+    class ReportingUrlChecker < AbstractUrlChecker
+
+      attr_reader :delegate
+
+      # Create a new instance.
+      # @param delegate_checker [AbstractUrlChecker].
+      def initialize(delegate_checker)
+        @logger = Logging.logger[self]
+        @delegate = delegate_checker
+      end
+
+      def check(*urls)
+        results = nil
+        benchmark = Benchmark.measure { results = @delegate.check(*urls) }
+        @logger.debug "Checking #{urls.size} URL(s) benchmark: #{benchmark}"
+
+        if urls.size > 1
+          @context = OpenStruct.new(
+              {
+                  benchmark: benchmark,
+                  results: results.sort_by { |status| status.uri }
+              })
+        else
+          @context = OpenStruct.new(
+              {
+                  benchmark: benchmark,
+                  results: [results]
+              })
+        end
+
+        results
+      end
+
+      # Creates a report about the previous check using the specified template. The result may be written to a file.
+      # @param template [Symbol, String] a predefined template (:csv, :html, :json) or the path of a template file.
+      # @param output_path [String, nil] the path of the output file.
+      # @return [String] the report.
+      def report(template, output_path=nil)
+        template_path = template_path(template)
+        template_content = ERB.new(File.new(template_path).read)
+        report = template_content.result(@context.instance_eval { binding })
+
+        if output_path
+          @logger.info "Reporting to #{output_path}"
+          File.open(output_path, 'w') { |file| file.write(report) }
+        end
+
+        report
+      end
+
+
+      # -------------------------------------------------------------------------------------------------- private -----
+      private
+
+      def template_path(symbol_or_path)
+        case symbol_or_path
+          when :csv then
+            # File.expand_path(File.join(File.dirname(__FILE__), 'reporting_url_checker.csv.erb'))
+            raise 'TODO'
+          when :html then
+            File.expand_path(File.join(File.dirname(__FILE__), 'reporting_url_checker.html.erb'))
+          when :json then
+            # File.expand_path(File.join(File.dirname(__FILE__), 'reporting_url_checker.json.erb'))
+            raise 'TODO'
+          else
+            symbol_or_path
+        end
+      end
+    end
+  end
+end