html-proofer 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2eb21756ab1eb24e38b51ada6d481b1dbc42eb8
-  data.tar.gz: bb779ed45d42bb8c9e7824cb66bcdd33f027c2ee
+  metadata.gz: 892dbf1a283cd5c3bf33d9a76430371d035f3abe
+  data.tar.gz: e4b6498ca79d5ad258f5763c005f039599304434
 SHA512:
-  metadata.gz: 3dd53810b0e3ae1b9760ced7d748100ab5ff56ed9b2b5f9770ea67960653b3a00d199a82f3a92c86c43054c3ec21191f82d9a860be389abce615aaf71b465425
-  data.tar.gz: 1225ae4c9a19292b05017409233dd8e8d1c0bcc5761038a8b40f62bebcc5926073e185ff7a4d4de205a4ffd53886d91481adaba54a57c13eba0086eeac404aba
+  metadata.gz: fc79438fbab4a5e7d7e9c9f318d3cfb71e315e6028a1b9b0e1934babb6a4aa046930ff830cf193baffec171785421d28fdbfb5470b6e8781a6ecaaed4eecfece
+  data.tar.gz: 45ff2420a45ebda37369dfcf713f4b9aa89ba3e8f4278c705345fa83915cf6fe5d76cf04b1a80b102fb4123f9c3ba365eceb4f0a808f66011f4f6665fc15807c

data/README.md

@@ -122,6 +122,11 @@ Project | Repository
 * Whether your internal script references are not broken
 * Whether external scripts are loading
 
+### HTML
+
+Nokogiri looks at the markup and [provides errors](http://www.nokogiri.org/tutorials/ensuring_well_formed_markup.html) when parsing your document.
+This is an optional feature, set the `validate_html` option to enable validation errors from Nokogiri.
+
 ## Configuration
 
 The `HTML::Proofer` constructor takes an optional hash of additional options:
@@ -133,11 +138,12 @@ The `HTML::Proofer` constructor takes an optional hash of additional options:
 | `favicon` | Enables the favicon checker. | `false` |
 | `followlocation` | Follows external redirections. Amends missing trailing slashes to internal directories. | `true` |
 | `directory_index_file` | Sets the file to look for when a link refers to a directory. | `index.html` |
-| `href_ignore` | An array of Strings or RegExps containing `href`s that are safe to ignore. Certain URIs, like `mailto` and `tel`, are always ignored. | `[]` |
+| `href_ignore` | An array of Strings or RegExps containing `href`s that are safe to ignore. Note that non-HTTP(S) URIs are always ignored. | `[]` |
 | `alt_ignore` | An array of Strings or RegExps containing `img`s whose missing `alt` tags are safe to ignore. | `[]` |
 | `href_swap` | A hash containing key-value pairs of `RegExp => String`. It transforms links that match `RegExp` into `String` via `gsub`. | `{}` |
 | `verbose` | If `true`, outputs extra information as the checking happens. Useful for debugging. | `false` |
 | `only_4xx` | Only reports errors for links that fall within the 4xx status code range. | `false` |
+| `validate_html` | Enables HTML validation errors from Nokogiri | `false` |
 
 ### Configuring Typhoeus
 
@@ -181,7 +187,7 @@ Add the `data-proofer-ignore` attribute to any tag to ignore it from the checks.
 
 Want to write your own test? Sure! Just create two classes--one that inherits from `HTML::Proofer::Checkable`, and another that inherits from `HTML::Proofer::Checks::Check`. `Checkable` defines various helper methods for your test, while `Checks::Check` actually runs across your content. `Checks::Check` should call `self.add_issue` on failures, to add them to the list.
 
-Here's an example custom test that protects against `mailto` links:
+Here's an example custom test that protects against `mailto` links that point to `octocat@github.com`:
 
 ``` ruby
 class OctocatLink < ::HTML::Proofer::Checkable

data/bin/htmlproof

@@ -18,12 +18,13 @@ Mercenary.program(:htmlproof) do |p|
   p.option 'favicon', '--favicon', 'Enables the favicon checker (default: `false`).'
   p.option 'as-links', '--as-links', 'Assumes that `PATH` is a comma-separated array of links to check.'
   p.option 'swap', '--swap regex:string,[regex:string,...]', Array, 'Array containing key-value pairs of `RegExp:String`. It transforms links that match `RegExp` into `String`'
-  p.option 'href_ignore', '--href_ignore link1,[link2,...]', Array, 'Array of Strings containing `href`s that are safe to ignore. Certain URIs, like `mailto` and `tel`, are always ignored.'
+  p.option 'href_ignore', '--href_ignore link1,[link2,...]', Array, 'Array of Strings containing `href`s that are safe to ignore. Note that non-HTTP(S) URIs are always ignored.'
   p.option 'alt_ignore', '--alt_ignore image1,[image2,...]', Array, 'Array of Strings containing `img`s whose missing `alt` tags are safe to ignore'
   p.option 'disable_external', '--disable_external', 'Disables the external link checker (default: `false`)'
   p.option 'only-4xx', '--only-4xx', 'Only reports errors for links that fall within the 4x status code range.'
   p.option 'verbose', '--verbose', 'Enables more verbose logging.'
   p.option 'directory_index_file', '--directory_index_file', 'Sets the file to look for when a link refers to a directory.'
+  p.option 'validate_html', '--validate_html', 'Enables HTML validation errors from Nokogiri (default: `false`).'
 
   p.action do |args, opts|
     args = ["."] if args.empty?
@@ -45,6 +46,7 @@ Mercenary.program(:htmlproof) do |p|
     options[:favicon] = opts["favicon"] unless opts["favicon"].nil?
     options[:verbose] = opts["verbose"] unless opts["verbose"].nil?
     options[:directory_index_file] = opts["directory_index_file"] unless opts["directory_index_file"].nil?
+    options[:validate_html] = opts["validate_html"] unless opts["validate_html"].nil?
 
     path = path.delete(' ').split(",") if opts["as-links"]
 

data/html-proofer.gemspec

@@ -1,9 +1,10 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
+require 'html/proofer/version'
 
 Gem::Specification.new do |gem|
   gem.name          = "html-proofer"
-  gem.version       = "1.4.0"
+  gem.version       = HTML::Proofer::VERSION
   gem.authors       = ["Garen Torikian"]
   gem.email         = ["gjtorikian@gmail.com"]
   gem.description   = %q{Test your rendered HTML files to make sure they're accurate.}
@@ -21,9 +22,10 @@ Gem::Specification.new do |gem|
   gem.add_dependency "typhoeus",        "~> 0.6.7"
   gem.add_dependency "yell",            "~> 2.0"
   gem.add_dependency "parallel",        "~> 1.3"
+  gem.add_dependency "addressable",     "~> 2.3"
 
   gem.add_development_dependency "redcarpet"
-  gem.add_development_dependency "rspec", "~> 2.13.0"
+  gem.add_development_dependency "rspec", "~> 3.1.0"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "awesome_print"
 end

data/lib/html/proofer.rb

@@ -1,15 +1,17 @@
 require 'nokogiri'
 require 'yell'
 require 'parallel'
+require "addressable/uri"
 
 begin
   require "awesome_print"
 rescue LoadError; end
 
-[
-  'checkable',
-  'checks',
-  'issue'
+%w[
+  checkable
+  checks
+  issue
+  version
 ].each { |r| require File.join(File.dirname(__FILE__), "proofer", r) }
 
 module HTML
@@ -39,7 +41,9 @@ module HTML
         :disable_external => false,
         :verbose => false,
         :only_4xx => false,
-        :directory_index_file => "index.html"
+        :directory_index_file => "index.html",
+        :validate_html => false,
+        :error_sort => :path
       }
 
       @typhoeus_opts = {
@@ -104,8 +108,35 @@ module HTML
       if @failed_tests.empty?
         logger.info HTML::colorize :green, "HTML-Proofer finished successfully."
       else
-        @failed_tests.sort_by(&:path).each do |issue|
-          logger.error HTML::colorize :red, issue.to_s
+        matcher = nil
+
+        # always sort by the actual option, then path, to ensure consistent alphabetical (by filename) results
+        @failed_tests = @failed_tests.sort do |a,b|
+          comp = (a.send(@options[:error_sort]) <=> b.send(@options[:error_sort]))
+          comp.zero? ? (a.path <=> b.path) : comp
+        end
+
+        @failed_tests.each do |issue|
+          case @options[:error_sort]
+          when :path
+            if matcher != issue.path
+              logger.error HTML::colorize :blue, "- #{issue.path}"
+              matcher = issue.path
+            end
+            logger.error HTML::colorize :red, "  *  #{issue.desc}"
+          when :desc
+            if matcher != issue.desc
+              logger.error HTML::colorize :blue, "- #{issue.desc}"
+              matcher = issue.desc
+            end
+            logger.error HTML::colorize :red, "  *  #{issue.path}"
+          when :status
+            if matcher != issue.status
+              logger.error HTML::colorize :blue, "- #{issue.status}"
+              matcher = issue.status
+            end
+            logger.error HTML::colorize :red, "  *  #{issue.to_s}"
+          end
         end
 
         raise HTML::colorize :red, "HTML-Proofer found #{@failed_tests.length} failures!"
@@ -125,7 +156,11 @@ module HTML
       Ethon.logger = logger # log from Typhoeus/Ethon
 
       external_urls.each_pair do |href, filenames|
-        queue_request(:head, href, filenames)
+        if has_hash? href
+          queue_request(:get, href, filenames)
+        else
+          queue_request(:head, href, filenames)
+        end
       end
       logger.debug HTML::colorize :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..."
       hydra.run
@@ -138,7 +173,8 @@ module HTML
     end
 
     def response_handler(response, filenames)
-      href = response.options[:effective_url]
+      effective_url = response.options[:effective_url]
+      href = response.request.base_url
       method = response.request.options[:method]
       response_code = response.code
 
@@ -147,7 +183,18 @@ module HTML
       logger.debug debug_msg
 
       if response_code.between?(200, 299)
-        # continue with no op
+        return if @options[:only_4xx]
+        if hash = has_hash?(href)
+          body_doc = Nokogiri::HTML(response.body)
+          # user-content is a special addition by GitHub.
+          if URI.parse(href).host.match(/github\.com/i)
+            if body_doc.xpath(%$//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"]$).empty?
+              add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response_code
+            end
+          elsif body_doc.xpath(%$//*[@name="#{hash}"]|//*[@id="#{hash}"]$).empty?
+            add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response_code
+          end
+        end
       elsif response.timed_out?
         return if @options[:only_4xx]
         add_failed_tests filenames, "External link #{href} failed: got a time out", response_code
@@ -189,9 +236,18 @@ module HTML
     def get_checks
       checks = HTML::Proofer::Checks::Check.subclasses.map { |c| c.name }
       checks.delete("Favicons") unless @options[:favicon]
+      checks.delete("Html") unless @options[:validate_html]
       checks
     end
 
+    def has_hash?(url)
+      begin
+        URI.parse(url).fragment
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+
     def log_level
       @options[:verbose] ? :debug : :info
     end

data/lib/html/proofer/check.rb

@@ -26,7 +26,7 @@ class HTML::Proofer::Checks
       raise NotImplementedError.new("HTML::Proofer::Check subclasses must implement #run")
     end
 
-    def add_issue(desc, status = nil)
+    def add_issue(desc, status = -1)
       @issues << Issue.new(@path, desc, status)
     end
 

data/lib/html/proofer/checkable.rb

@@ -36,11 +36,9 @@ module HTML
       end
 
       def parts
-        return @parts_cached if defined?(@parts_cached)
-
-        @parts_cached = URI::Parser.new(:ESCAPED => '\%|\|').parse url
+        @parts ||= Addressable::URI.parse url
       rescue URI::Error
-        @parts_cached = nil
+        @parts = nil
       end
 
       def path
@@ -60,6 +58,10 @@ module HTML
         %w( http https ).include? scheme
       end
 
+      def non_http_remote?
+        !scheme.nil? && !remote?
+      end
+
       def ignore?
         return true if @data_ignore_proofer
 
@@ -72,8 +74,6 @@ module HTML
           return true if url.match(/^data:image/)
           return true if ignores_pattern_check(@check.additional_alt_ignores)
         end
-
-        %w( mailto tel ).include? scheme
       end
 
       # path is external to the file

data/lib/html/proofer/checks.rb

@@ -7,7 +7,8 @@ module HTML
         "checks/images",
         "checks/links",
         "checks/scripts",
-        "checks/favicon"
+        "checks/favicon",
+        "checks/html"
       ].each { |r| require File.join(File.dirname(__FILE__), r) }
     end
   end

data/lib/html/proofer/checks/favicon.rb

@@ -9,8 +9,6 @@ end
 class Favicons < ::HTML::Proofer::Checks::Check
 
   def run
-    return unless @options[:favicon]
-
     @html.xpath("//link[not(ancestor::pre or ancestor::code)]").each do |favicon|
       favicon = Favicon.new favicon, "favicon", self
       next if favicon.ignore?

data/lib/html/proofer/checks/html.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+class Html < ::HTML::Proofer::Checks::Check
+
+  # new html5 tags (source: http://www.w3schools.com/html/html5_new_elements.asp)
+  HTML5_TAGS = %w(article aside bdi details dialog figcaption 
+                  figure footer header main mark menuitem meter 
+                  nav progress rp rt ruby section summary 
+                  time wbr datalist keygen output color date 
+                  datetime datetime-local email month number 
+                  range search tel time url week canvas 
+                  svg audio embed source track video)
+
+  def run
+    @html.errors.each do |e|
+
+        # Nokogiri (or rather libxml2 underhood) only recognizes html4 tags,
+        # so we need to skip errors caused by the new tags in html5
+        next if HTML5_TAGS.include? e.to_s[/Tag ([\w-]+) invalid/o, 1]
+
+        self.add_issue(e.to_s)
+    end
+  end
+end

data/lib/html/proofer/checks/links.rb

@@ -46,12 +46,24 @@ class Links < ::HTML::Proofer::Checks::Check
         next
       end
 
+      if link.scheme == "mailto"
+        self.add_issue "#{link.href} contains no email address" if link.path.empty?
+        self.add_issue "#{link.href} contain an invalid email address" unless link.path.include?("@")
+      end
+
+      if link.scheme == "tel"
+        self.add_issue "#{link.href} contains no phone number" if link.path.empty?
+      end
+
       # is there even a href?
       if link.missing_href?
         self.add_issue("anchor has no href attribute")
         next
       end
 
+      # intentionally here because we still want valid? & missing_href? to execute
+      next if link.non_http_remote?
+
       # does the file even exist?
       if link.remote?
         add_to_external_urls link.href

data/lib/html/proofer/issue.rb

@@ -7,7 +7,7 @@ class HTML::Proofer::Checks
 
     attr_reader :path, :desc, :status
 
-    def initialize(path, desc, status = nil)
+    def initialize(path, desc, status = -1)
       @path = path
       @desc = desc
       @status = status

data/lib/html/proofer/version.rb

@@ -0,0 +1,5 @@
+module HTML
+  class Proofer
+    VERSION = "1.5.0"
+  end
+end

data/spec/html/proofer/favicon_spec.rb

@@ -3,44 +3,45 @@ require "spec_helper"
 describe "Favicons test" do
   it "ignores for absent favicon by default" do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent.html"
-    output = capture_stderr { HTML::Proofer.new(absent).run }
-    output.should == ""
+    expect(make_proofer(absent).failed_tests).to eq []
   end
 
   it "fails for absent favicon" do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent.html"
-    output = capture_stderr { HTML::Proofer.new(absent, {:favicon => true}).run }
-    output.should match /no favicon specified/
+    proofer = make_proofer(absent, {:favicon => true})
+    expect(proofer.failed_tests.first).to match /no favicon specified/
   end
 
   it "fails for absent favicon but present apple touch icon" do
     absent = "#{FIXTURES_DIR}/favicon/favicon_absent_apple.html"
-    output = capture_stderr { HTML::Proofer.new(absent, {:favicon => true}).run }
-    output.should match /no favicon specified/
+    proofer = make_proofer(absent, {:favicon => true})
+    # Travis gives a different error message here for some reason
+    expect(proofer.failed_tests.last).to match /(internally linking to gpl.png, which does not exist|no favicon specified)/
   end
 
   it "fails for broken favicon" do
     broken = "#{FIXTURES_DIR}/favicon/favicon_broken.html"
-    output = capture_stderr { HTML::Proofer.new(broken, {:favicon => true}).run }
-    output.should match /internally linking to asdadaskdalsdk.png/
+    proofer = make_proofer(broken, {:favicon => true})
+
+    expect(proofer.failed_tests.first).to match /internally linking to asdadaskdalsdk.png/
   end
 
   it "passes for present favicon" do
     present = "#{FIXTURES_DIR}/favicon/favicon_present.html"
-    output = capture_stderr { HTML::Proofer.new(present, {:favicon => true}).run }
-    output.should == ""
+    proofer = make_proofer(present, {:favicon => true})
+    expect(proofer.failed_tests).to eq []
   end
 
   it "passes for present favicon with shortcut notation" do
     present = "#{FIXTURES_DIR}/favicon/favicon_present_shortcut.html"
-    output = capture_stderr { HTML::Proofer.new(present, {:favicon => true}).run }
-    output.should == ""
+    proofer =  make_proofer(present, {:favicon => true})
+    expect(proofer.failed_tests).to eq []
   end
 
   it "fails for broken favicon with data-proofer-ignore" do
     broken_but_ignored = "#{FIXTURES_DIR}/favicon/favicon_broken_but_ignored.html"
-    output = capture_stderr { HTML::Proofer.new(broken_but_ignored, {:favicon => true}).run }
-    output.should match /no favicon specified/
+    proofer = make_proofer(broken_but_ignored, {:favicon => true})
+    expect(proofer.failed_tests.first).to match /no favicon specified/
   end
 
 end

data/spec/html/proofer/fixtures/html/div_inside_head.html

@@ -0,0 +1,6 @@
+<html>
+<head>
+<div>
+</div>
+</head>
+</html>

data/spec/html/proofer/fixtures/html/html5_tags.html

@@ -0,0 +1,9 @@
+<html>
+<body>
+<nav>
+</nav>
+<article>
+<p>Some text</p>
+</article>
+</body>
+</html>

data/spec/html/proofer/fixtures/html/invalid_tag.html

@@ -0,0 +1,3 @@
+<html>
+<myfancytag></myfancytag>
+</html>

data/spec/html/proofer/fixtures/html/missing_closing_quotes.html

@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="/test>A test link</a>
+</body>
+</html>