ndd-url_checker 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ddb514dd4940f3ca4e1c12b26d11569a5d14196
-  data.tar.gz: def79c9bdbfaa493381bcc6bce5e147363afba86
+  metadata.gz: 6e878e5f4aff0bed140d52f9e1c888d9c951fc95
+  data.tar.gz: 1e5754bb7e356951ab4798083eada1c2b227a2a1
 SHA512:
-  metadata.gz: 46cc5e1d3a0442608626e1d86d7e77730b8a5f012781a7711fd7de39f88999e7d776137b9493dcc2117f28db7636b9f439a7c773fcc383c6b5f50e40bdef6e10
-  data.tar.gz: de7f6677ed2b53345150cbb1eb71cc8ee0608957bb28b31f273e43af58dd5b50a68b8794cfab2a48229bcec4f593f4ad297d82d740d50705d034b3337e5bdaa0
+  metadata.gz: 8e93630400c83f45833701355144aee71a907b78852bc2dde095efdc17598e621de1d19d5ffbf8e57cef66f1df289b4674e00e95ab155d7e79874cd8e585edd4
+  data.tar.gz: 2c7228f881743e6f8adc6bbd5f2908f1e0d8da0d05c69ffa9d000952c7618b89841e91740d52ffa436af64f39322b6e008250e67604ac6c9a11e28cefca2ead9

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.5
+ruby-2.2.2

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 rvm:
   - 2.0.0
-  - 2.1.5
+  - 2.1.6
+  - 2.2.2
 before_install:
   - gem update --system
   - gem --version

data/Gemfile.lock

@@ -1,14 +1,14 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.3.6)
+    addressable (2.3.8)
     builder (3.2.2)
     celluloid (0.16.0)
       timers (~> 4.0.0)
-    childprocess (0.5.5)
+    childprocess (0.5.6)
       ffi (~> 1.0, >= 1.0.11)
     cod (0.6.0)
-    codeclimate-test-reporter (0.4.3)
+    codeclimate-test-reporter (0.4.7)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.0)
     crack (0.4.2)
@@ -17,12 +17,12 @@ GEM
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    faraday (0.9.0)
+    faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
-    ffi (1.9.6)
+    ffi (1.9.8)
     formatador (0.2.5)
-    git (1.2.8)
-    github_api (0.12.2)
+    git (1.2.9.1)
+    github_api (0.12.3)
       addressable (~> 2.3)
       descendants_tracker (~> 0.0.4)
       faraday (~> 0.8, < 0.10)
@@ -30,26 +30,31 @@ GEM
       multi_json (>= 1.7.5, < 2.0)
       nokogiri (~> 1.6.3)
       oauth2
-    guard (2.8.2)
+    guard (2.12.6)
       formatador (>= 0.2.4)
       listen (~> 2.7)
       lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
       pry (>= 0.9.12)
+      shellany (~> 0.0)
       thor (>= 0.18.1)
-    guard-bundler (2.0.0)
+    guard-bundler (2.1.0)
       bundler (~> 1.0)
       guard (~> 2.2)
-    guard-compat (0.3.0)
-    guard-rspec (4.4.2)
+      guard-compat (~> 1.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.5.2)
       guard (~> 2.1)
-      guard-compat (~> 0.1)
+      guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    guard-spork (2.0.1)
+    guard-spork (2.1.0)
       childprocess (>= 0.2.3)
-      guard (~> 2.8.2)
+      guard (~> 2.0)
+      guard-compat (~> 1.0)
       spork (>= 0.8.4)
-    hashie (3.3.2)
-    highline (1.6.21)
+    hashie (3.4.2)
+    highline (1.7.2)
     hitimes (1.2.2)
     jeweler (2.0.1)
       builder
@@ -60,12 +65,12 @@ GEM
       nokogiri (>= 1.5.10)
       rake
       rdoc
-    json (1.8.1)
-    jwt (1.2.0)
-    libnotify (0.8.4)
+    json (1.8.3)
+    jwt (1.5.0)
+    libnotify (0.9.1)
       ffi (>= 1.0.11)
-    listen (2.8.3)
-      celluloid (>= 0.15.2)
+    listen (2.10.0)
+      celluloid (~> 0.16.0)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
     little-plugger (1.1.3)
@@ -74,12 +79,16 @@ GEM
       multi_json (>= 1.8.4)
     lumberjack (1.0.9)
     method_source (0.8.2)
-    mini_portile (0.6.1)
-    multi_json (1.10.1)
+    mini_portile (0.6.2)
+    multi_json (1.11.0)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
-    nokogiri (1.6.5)
+    nenv (0.2.0)
+    nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
+    notiffany (0.0.6)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
     oauth2 (1.0.0)
       faraday (>= 0.8, < 0.10)
       jwt (~> 1.0)
@@ -90,40 +99,41 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rack (1.5.2)
+    rack (1.6.1)
     rake (10.4.2)
-    rb-fsevent (0.9.4)
+    rb-fsevent (0.9.5)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
     rdoc (4.2.0)
-      json (~> 1.4)
-    rspec (3.1.0)
-      rspec-core (~> 3.1.0)
-      rspec-expectations (~> 3.1.0)
-      rspec-mocks (~> 3.1.0)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
     rspec-collection_matchers (1.1.2)
       rspec-expectations (>= 2.99.0.beta1)
-    rspec-core (3.1.7)
-      rspec-support (~> 3.1.0)
-    rspec-expectations (3.1.2)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.1.0)
-    rspec-mocks (3.1.3)
-      rspec-support (~> 3.1.0)
-    rspec-support (3.1.2)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
     safe_yaml (1.0.4)
-    simplecov (0.9.1)
+    shellany (0.0.1)
+    simplecov (0.10.0)
       docile (~> 1.1.0)
-      multi_json (~> 1.0)
-      simplecov-html (~> 0.8.0)
-    simplecov-html (0.8.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
     slop (3.6.0)
     spork (0.9.2)
     thor (0.19.1)
-    thread_safe (0.3.4)
+    thread_safe (0.3.5)
     timers (4.0.1)
       hitimes
-    webmock (1.20.4)
+    webmock (1.21.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
     yard (0.8.7.6)
@@ -150,3 +160,6 @@ DEPENDENCIES
   spork (~> 0.9)
   webmock (~> 1.20)
   yard (~> 0.8)
+
+BUNDLED WITH
+   1.10.3

data/README.md

@@ -14,7 +14,8 @@ The API documentation can be find at [RubyDoc](http://www.rubydoc.info/github/dd
 This gem requires Ruby 2.x and is tested with:
 
 - Ruby 2.0.0
-- Ruby 2.1.x
+- Ruby 2.1.6
+- Ruby 2.2.2
 
 ## Usage
 
@@ -64,12 +65,14 @@ checker.check('http://www.google.com/')
 
 ### ReportingUrlChecker
 
-For a nice looking report, use [`NDD::UrlChecker::ReportingUrlChecker`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/ReportingUrlChecker) which enables reporting capabilities using ERB templates. Several built-in templates are provided: CSV (planed), HTML and JSON (planed).
+For a nice looking report, use [`NDD::UrlChecker::ReportingUrlChecker`](http://www.rubydoc.info/github/ddidier/ndd-url_checker/NDD/UrlChecker/ReportingUrlChecker) which enables reporting capabilities using ERB templates. Several built-in templates are provided: CSV, HTML and JSON.
 
 ```ruby
 checker = NDD::UrlChecker:: ReportingUrlChecker.new(delegate_checker)
 statuses = checker.check('http://www.invalid123456789.com/', 'http://www.google.com/')
+report_as_text = checker.report(:csv, '/some/report.csv')
 report_as_text = checker.report(:html, '/some/report.html')
+report_as_text = checker.report(:json, '/some/report.json')
 report_as_text = checker.report('/some/template.erb', '/some/report.html')
 ```
 

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2a

data/lib/ndd/url_checker.rb

@@ -2,5 +2,7 @@ require 'ndd/url_checker/abstract_url_checker'
 require 'ndd/url_checker/blocking_url_checker'
 require 'ndd/url_checker/forked_url_checker'
 require 'ndd/url_checker/parallel_url_checker'
+require 'ndd/url_checker/reporting_url_checker'
 require 'ndd/url_checker/status'
+require 'ndd/url_checker/status_decorator'
 require 'ndd/url_checker/threaded_url_checker'

data/lib/ndd/url_checker/blocking_url_checker.rb

@@ -25,7 +25,7 @@ module NDD
       # @return [NDD::UrlChecker::Status, Array<NDD::UrlChecker::Status>] a single status for a single URL, an array
       #         of status for multiple parameters
       def check(*urls)
-        @logger.info "Checking #{urls.size} URL(s)"
+        @logger.debug "Checking #{urls.size} URL(s)"
         return check_single(urls.first) if urls.size < 2
         urls.map { |url| check_single(url) }
       end
@@ -40,12 +40,13 @@ module NDD
       def check_single(url)
         begin
           @logger.debug "Checking: #{url}"
-          status = check_uri(URI.parse(url), Status.new(url))
+          status = Status.new(url)
+          status = check_uri(URI.parse(url), status)
         rescue => error
           status = if unknown_host?(error)
-                     Status.new(url).unknown_host
+                     status.unknown_host
                    else
-                     Status.new(url).failed(error)
+                     status.failed(error)
                    end
         end
         @logger.debug "Checked: #{url} -> #{status.code.upcase}"
@@ -91,6 +92,8 @@ module NDD
       end
 
       def on_error(uri, response, status)
+        # read the body while the socket is still open
+        response.body if response.kind_of?(Net::HTTPResponse)
         status.failed(response)
       end
 

data/lib/ndd/url_checker/forked_url_checker.rb

@@ -42,9 +42,11 @@ module NDD
         end
 
         # read back the results
-        results = urls.map do |_|
+        results = urls.each_with_index.map do |_, index|
           result = result_pipe.get
-          @logger.debug("Processed URLs #{hash.size}/#{urls.size}")
+          processed_number = index + 1
+          processed_percentage = processed_number.to_f / urls.size.to_f * 100.0
+          @logger.debug("Processed URLs #{processed_number}/#{urls.size} (#{processed_percentage.round(2)} %)")
           result
         end
 

data/lib/ndd/url_checker/reporting_url_checker.csv.erb

@@ -1,4 +1,4 @@
-Index,URL,Result,Code,Comment
-<% results.each_with_index do |status, index| %>
+Index,URL,Result,Code,Details
+<% statuses.each_with_index do |status, index| %>
 <%= index+1 %>,<%= status.uri %>,<%= status.valid? ? 'valid' : 'invalid' %>,<%= status.code %>,<% if status.invalid? %><%= status.error %><% end %>
 <% end %>

data/lib/ndd/url_checker/reporting_url_checker.html.erb

@@ -2,96 +2,187 @@
 <html lang="en">
 
 <head>
-  <meta charset="UTF-8">
-  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
-  <style>
-    body {
-      padding: 1em;
-    }
+  <meta charset="UTF-8" />
+
+  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css" />
+  <style type="text/css">
+      .popover-title {
+          font-weight: bold;
+      }
+
+      .popover-content em {
+          font-style: normal;
+          font-weight: bold;
+      }
+
+      .popover-content ol {
+          margin-left: -20px;
+      }
   </style>
 
-  <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
-  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js"></script>
+  <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
+  <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
   <script language="JavaScript">
-    $(document).ready(function() {
-      $('input[type=radio][name=urlsFilter]').change(function() {
-        if (this.value == 'all') {
-          $('tr').show();
-        }
-        else if (this.value == 'invalid') {
-          $('tr').not('.danger').hide();
-        }
+      if (!window.ndd_url_checker) {
+          window.ndd_url_checker = {};
+      }
+
+      ndd_url_checker.STATES = ['direct', 'redirected', 'failed', 'too-many-redirects', 'unknown-host'];
+
+      ndd_url_checker.displayStates = function () {
+          var displayStates = {};
+          $.each(ndd_url_checker.STATES, function (_, state) {
+              var checked = $('.display-' + state).is(':checked');
+              displayStates[state] = checked;
+          });
+          return displayStates;
+      }
+
+      ndd_url_checker.filter = function () {
+          var displayStates = ndd_url_checker.displayStates();
+
+          $.each(displayStates, function (state, display) {
+              $('.result-' + state).toggle(display);
+          });
+      }
+
+      $(document).ready(function () {
+          // show or hide results depending on their state
+          $.each(ndd_url_checker.STATES, function (_, state) {
+              $('.display-' + state).change(function () {
+                  ndd_url_checker.filter();
+              });
+          });
+
+          // activate all tooltips
+          $('[data-toggle="popover"]')
+                  .popover()
+                  .click(function (e) {
+                      e.preventDefault();
+                  });
       });
-    });
   </script>
 </head>
 
+
 <body>
 
-<h1>URL Checker Report</h1>
-
-<table class="table table-condensed table-hover" style="width: auto">
-  <tbody>
-  <tr>
-    <td>User CPU time</td>
-    <td><%= benchmark.utime %> second(s)</td>
-  </tr>
-  <tr>
-    <td>System CPU time</td>
-    <td><%= benchmark.stime %> second(s)</td>
-  </tr>
-  <tr>
-    <td>Total time</td>
-    <td><%= benchmark.total %> second(s)</td>
-  </tr>
-  <tr>
-    <td>Elapsed real time</td>
-    <td><%= benchmark.real %> second(s)</td>
-  </tr>
-  <tr>
-    <td>Number of URLs</td>
-    <td><%= results.size %></td>
-  </tr>
-  <tr>
-    <td>Average time for 1 URL</td>
-    <td><%= benchmark.real/results.size %> second(s)</td>
-  </tr>
-  </tbody>
-</table>
-
-<div class="radio">
-  <label>
-    <input type="radio" name="urlsFilter" id="showAllUrls" value="all" checked="true" />
-    Show all URLs
-  </label>
-</div>
-<div class="radio">
-  <label>
-    <input type="radio" name="urlsFilter" id="showInvalidUrls" value="invalid" />
-    Show invalid URLs
-  </label>
-</div>
+<div class="container">
+
+  <div class="page-header">
+    <h1>
+      <span class="glyphicon glyphicon-list-alt" aria-hidden="true"></span>
+      URL Checker Report
+    </h1>
+  </div>
+
+
+  <div class="row">
+    <div class="col-md-4">
+      <dl class="page-info dl-horizontal">
+        <dt>Number of URLs</dt>
+        <dd><%= urls.count %></dd>
+        <dt>Duration</dt>
+        <dd><%= benchmark.total_duration.round(3) %> s</dd>
+        <dt>Average duration</dt>
+        <dd><%= benchmark.average_duration.round(3) %> s/URL</dd>
+        <dt>Average throughput</dt>
+        <dd><%= benchmark.average_throughput.round(3) %> URL/s</dd>
+      </dl>
+    </div>
+    <div class="col-md-4">
+      <dl class="page-info dl-horizontal">
+        <dt>Valid URLS</dt>
+        <dd><%= urls.valid_count %></dd>
+        <dt>direct</dt>
+        <dd><%= urls.direct_count %></dd>
+        <dt>redirected</dt>
+        <dd><%= urls.redirected_count %></dd>
+      </dl>
+    </div>
+    <div class="col-md-4">
+      <dl class="page-info dl-horizontal">
+        <dt>Invalid URLS</dt>
+        <dd><%= urls.invalid_count %></dd>
+        <dt>failed</dt>
+        <dd><%= urls.failed_count %></dd>
+        <dt>too many redirects</dt>
+        <dd><%= urls.too_many_redirects_count %></dd>
+        <dt>unknown host</dt>
+        <dd><%= urls.unknown_host_count %></dd>
+      </dl>
+    </div>
+  </div>
+  <hr />
+
+
+  <div class="row page-controls hidden-print">
+    <div class="col-md-6">
+      <span>Valid URLs:</span>
 
-<table class="table table-condensed table-hover" style="width: auto">
-  <thead>
-  <tr>
-    <th>#</th>
-    <th>URL</th>
-    <th>Code</th>
-    <th>Comment</th>
-  </tr>
-  </thead>
-  <tbody>
-  <% results.each_with_index do |status, index| %>
-      <tr class="<%= status.invalid? ? 'danger' : '' %>">
-        <td><%= index+1 %></td>
-        <td><a href="<%= status.uri %>" target="_blank"><%= status.uri %></a></td>
-        <td><%= status.code %></td>
-        <td><% if status.invalid? %><%= status.error %><% end %></td>
-      </tr>
-  <% end %>
-  </tbody>
-</table>
+      <div class="btn-group page-display" data-toggle="buttons">
+        <label class="btn btn-default active">
+          <input type="checkbox" autocomplete="off" class="display-direct" checked="true" />
+          Direct (<%= urls.direct_count %>)
+        </label>
+        <label class="btn btn-default active">
+          <input type="checkbox" autocomplete="off" class="display-redirected" checked="true" />
+          Redirected (<%= urls.redirected_count %>)
+        </label>
+      </div>
+    </div>
+    <div class="col-md-6">
+      <span>Invalid URLs:</span>
+
+      <div class="btn-group page-display" data-toggle="buttons">
+        <label class="btn btn-default active">
+          <input type="checkbox" autocomplete="off" class="display-failed" checked="true" />
+          Failed (<%= urls.failed_count %>)
+        </label>
+        <label class="btn btn-default active">
+          <input type="checkbox" autocomplete="off" class="display-too-many-redirects" checked="true" />
+          Too many redirects (<%= urls.too_many_redirects_count %>)
+        </label>
+        <label class="btn btn-default active">
+          <input type="checkbox" autocomplete="off" class="display-unknown-host" checked="true" />
+          Unknown host (<%= urls.unknown_host_count %>)
+        </label>
+      </div>
+    </div>
+  </div>
+  <hr />
+
+
+  <table id="results" class="table table-condensed table-hover">
+    <thead>
+    <tr>
+      <th>#</th>
+      <th>URL</th>
+      <th>Code</th>
+      <th>Details</th>
+    </tr>
+    </thead>
+    <tbody>
+    <% statuses.each_with_index do |status, index| %>
+        <tr class="result result-<%= status.code_as_css %> <%= status.invalid? ? 'danger' : '' %>">
+          <td><%= index+1 %></td>
+          <td><a href="<%= status.uri %>" target="_blank"><%= status.uri %></a></td>
+          <td><%= status.code %></td>
+          <td><%= status.details_title %></td>
+          <td>
+            <% if status.details_body %>
+                <a href="#" data-toggle="popover" data-placement="left" data-trigger="focus"
+                   data-html="true" title="<%= status.uri %>" data-content="<%= status.details_body_as_html %>">
+                  <span class="glyphicon glyphicon-info-sign" aria-hidden="true"></span>
+                </a>
+            <% end %>
+          </td>
+        </tr>
+    <% end %>
+    </tbody>
+  </table>
+
+</div>
 
 </body>
 </html>