validate-website 1.8.1 → 1.10.0

data/lib/validate_website/runner.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'validate_website/core'
 
 module ValidateWebsite
@@ -5,7 +7,7 @@ module ValidateWebsite
   class Runner
     def self.trap_interrupt
       trap('INT') do
-        STDERR.puts "\nExiting..."
+        warn "\nExiting..."
         exit!(1)
       end
     end

data/lib/validate_website/static.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'validate_website/core'
 require 'validate_website/utils'
 
@@ -5,12 +7,14 @@ module ValidateWebsite
   # Class for validation Static website
   class Static < Core
     CONTENT_TYPES = ['text/html', 'text/xhtml+xml'].freeze
+    START_MESSAGE_TYPE = 'files'
 
     attr_reader :history_count
 
     def initialize(options = {}, validation_type = :static)
       @history_count = 0
       super
+      start_message("#{START_MESSAGE_TYPE} in #{Dir.pwd} (#{@site} as site)")
     end
 
     # @param [Hash] options
@@ -20,11 +24,12 @@ module ValidateWebsite
       @site = @options[:site]
 
       files = Dir.glob(@options[:pattern])
-      files.each do |f|
-        next unless File.file?(f)
-        next if @options[:exclude].is_a?(Regexp) && @options[:exclude].match(f)
+      files.each do |file|
+        next unless File.file?(file)
+        next if @options[:exclude]&.match(file)
+
         @history_count += 1
-        check_static_file(f)
+        check_static_file(file)
       end
       print_status_line(files.size, 0, @not_founds_count, @errors_count)
     end
@@ -47,15 +52,18 @@ module ValidateWebsite
 
     private
 
-    def check_static_file(f)
-      page = StaticLink.new(f, @site).page
-      check_page(f, page)
+    def check_static_file(file)
+      page = StaticLink.new(file, @site).page
+      check_page(file, page)
       check_css_syntax(page) if page.css? && options[:css_syntax]
     end
 
-    def check_page(f, page)
+    def check_page(file, page)
       if page.html? && options[:markup]
-        validate(page.doc, page.body, f, options[:ignore])
+        keys = %i[ignore html5_validator]
+        # slice does not exists on Ruby <= 2.4
+        slice = Hash[[keys, options.values_at(*keys)].transpose]
+        validate(page.doc, page.body, file, slice)
       end
       check_static_not_found(page.links) if options[:not_found]
     end
@@ -66,11 +74,13 @@ module ValidateWebsite
       static_links = links.map { |l| StaticLink.new(l, @site) }
       static_links.each do |static_link|
         next unless static_link.check?
+
         unless File.exist?(static_link.file_path)
           not_found_error(static_link.file_path)
           next
         end
         next unless static_link.css?
+
         check_static_not_found static_link.extract_urls_from_fake_css_response
       end
     end

data/lib/validate_website/static_link.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'validate_website/utils'
 require 'validate_website/static'
@@ -25,9 +27,9 @@ StaticLink = Struct.new(:link, :site) do
 
   def body
     if File.exist?(link)
-      open(link).read
+      File.open(link).read
     else
-      open(file_path).read
+      File.open(file_path).read
     end
   end
 

data/lib/validate_website/utils.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Base module ValidateWebsite
 module ValidateWebsite
   # Utils class for CSS helpers
@@ -10,6 +12,7 @@ module ValidateWebsite
     def self.extract_urls_from_css(page)
       return Set[] unless page
       return Set[] if page.body.nil?
+
       nodes = Crass::Parser.parse_stylesheet(page.body)
       extract_urls_from_nodes nodes, page
     end

data/lib/validate_website/validator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 
 require 'nokogiri'
@@ -10,27 +12,25 @@ module ValidateWebsite
   class Validator
     extend ValidatorClassMethods
 
-    @html5_validator_service_url = 'https://checker.html5.org/'
+    @html5_validator_service_url = 'https://validator.nu/'
+    XHTML_SCHEMA_PATH = File.expand_path('../../data/schemas', __dir__)
+    @mutex = Mutex.new
+
     class << self
       attr_accessor :html5_validator_service_url
-    end
 
-    XHTML_PATH = File.expand_path('../../../data/schemas', __FILE__)
-
-    @xsd_schemas ||= {}
-
-    # `Dir.chdir` is needed by `Nokogiri::XML::Schema` to validate with local
-    # files and cannot use file absolute path.
-    Dir.glob(File.join(XHTML_PATH, '*.xsd')).each do |schema|
-      Dir.chdir(XHTML_PATH) do
-        schema_name = File.basename(schema, '.xsd')
-        schema_content = File.read(File.basename(schema))
-        begin
-          @xsd_schemas[schema_name] = Nokogiri::XML::Schema(schema_content)
-        rescue Nokogiri::XML::SyntaxError
-          STDERR.puts "XSD SCHEMA: #{schema} cannot be loaded"
+      # http://www.w3.org/TR/xhtml1-schema/
+      def schema(namespace)
+        @mutex.synchronize do
+          Dir.chdir(XHTML_SCHEMA_PATH) do
+            if File.exist?("#{namespace}.xsd")
+              Nokogiri::XML::Schema(File.read("#{namespace}.xsd"))
+            end
+          end
         end
       end
+
+      alias xsd schema
     end
 
     attr_reader :original_doc, :body, :dtd, :doc, :namespace, :html5_validator
@@ -65,25 +65,22 @@ module ValidateWebsite
       @ignore ? @errors.reject { |e| @ignore =~ e } : @errors
     end
 
-    # http://www.w3.org/TR/xhtml1-schema/
-    def self.xsd(namespace)
-      return unless namespace
-      @xsd_schemas[namespace] if @xsd_schemas.key? namespace
-    end
-
     private
 
     # http://www.w3.org/TR/xhtml1/#dtds
     def find_namespace(dtd)
       return unless dtd.system_id
+
       dtd_uri = URI.parse(dtd.system_id)
       return unless dtd_uri.path
+
       @dtd_uri = dtd_uri
       File.basename(@dtd_uri.path, '.dtd')
     end
 
     def document
       return @document if @document
+
       @document = if @dtd_uri && @body.match(@dtd_uri.to_s)
                     @body.sub(@dtd_uri.to_s, @namespace + '.dtd')
                   else
@@ -95,11 +92,11 @@ module ValidateWebsite
     def validate
       if document =~ /^\<!DOCTYPE html\>/i
         html5_validate
-      elsif self.class.xsd(@namespace)
-        self.class.xsd(@namespace).validate(xhtml_doc)
+      elsif self.class.schema(@namespace)
+        self.class.schema(@namespace).validate(xhtml_doc)
       else
         # dont have xsd fall back to dtd
-        Dir.chdir(XHTML_PATH) do
+        Dir.chdir(XHTML_SCHEMA_PATH) do
           Nokogiri::HTML.parse(document)
         end.errors
       end
@@ -138,8 +135,8 @@ module ValidateWebsite
     end
 
     def xhtml_doc
-      Dir.chdir(XHTML_PATH) do
-        Nokogiri::XML(document) { |cfg| cfg.noent.dtdload.dtdvalid }
+      Dir.chdir(XHTML_SCHEMA_PATH) do
+        Nokogiri::XML(document) { |cfg| cfg.nonoent.dtdload.dtdvalid.nonet }
       end
     end
   end

data/lib/validate_website/validator_class_methods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tidy_ffi'
 
 # Validator Class Methods
@@ -9,6 +11,7 @@ module ValidatorClassMethods
 
   def tidy
     return @tidy if defined?(@tidy)
+
     @lib_tidy = TidyFFI::LibTidy
     @tidy = TidyFFI::Tidy
   rescue TidyFFI::LibTidyNotInstalled

data/lib/validate_website/version.rb

@@ -2,5 +2,5 @@
 
 # Version file for ValidateWebsite
 module ValidateWebsite
-  VERSION = '1.8.1'.freeze
+  VERSION = '1.10.0'
 end

data/man/man1/validate-website-static.1

@@ -1,23 +1,32 @@
 '\" t
 .\"     Title: validate-website-static
-.\"    Author: [see the "AUTHORS" section]
-.\" Generator: Asciidoctor 1.5.6.1
-.\"      Date: 2018-01-20
+.\"    Author: [see the "AUTHOR(S)" section]
+.\" Generator: Asciidoctor 1.5.8
+.\"      Date: 2019-01-09
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE\-STATIC" "1" "2018-01-20" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE\-STATIC" "1" "2019-01-09" "\ \&" "\ \&"
 .ie \n(.g .ds Aq \(aq
 .el       .ds Aq '
 .ss \n[.ss] 0
 .nh
 .ad l
 .de URL
-\\$2 \(laURL: \\$1 \(ra\\$3
+\fI\\$2\fP <\\$1>\\$3
 ..
-.if \n[.g] .mso www.tmac
-.LINKSTYLE blue R < >
+.als MTO URL
+.if \n[.g] \{\
+.  mso www.tmac
+.  am URL
+.    ad l
+.  .
+.  am MTO
+.    ad l
+.  .
+.  LINKSTYLE blue R < >
+.\}
 .SH "NAME"
 validate\-website\-static \- check the validity of your documents
 .SH "SYNOPSIS"
@@ -119,5 +128,5 @@ Laurent Arnoud \c
 .sp
 The MIT License
 .sp
-Copyright (c) 2009\-2018 \c
+Copyright (c) 2009\-2019 \c
 .MTO "laurent\(atspkdev.net" "Laurent Arnoud" ""

data/man/man1/validate-website.1

@@ -1,23 +1,32 @@
 '\" t
 .\"     Title: validate-website
-.\"    Author: [see the "AUTHORS" section]
-.\" Generator: Asciidoctor 1.5.6.1
-.\"      Date: 2018-01-20
+.\"    Author: [see the "AUTHOR(S)" section]
+.\" Generator: Asciidoctor 1.5.8
+.\"      Date: 2019-01-09
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE" "1" "2018-01-20" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE" "1" "2019-01-09" "\ \&" "\ \&"
 .ie \n(.g .ds Aq \(aq
 .el       .ds Aq '
 .ss \n[.ss] 0
 .nh
 .ad l
 .de URL
-\\$2 \(laURL: \\$1 \(ra\\$3
+\fI\\$2\fP <\\$1>\\$3
 ..
-.if \n[.g] .mso www.tmac
-.LINKSTYLE blue R < >
+.als MTO URL
+.if \n[.g] \{\
+.  mso www.tmac
+.  am URL
+.    ad l
+.  .
+.  am MTO
+.    ad l
+.  .
+.  LINKSTYLE blue R < >
+.\}
 .SH "NAME"
 validate\-website \- Web crawler for checking the validity of your documents
 .SH "SYNOPSIS"
@@ -124,5 +133,5 @@ Laurent Arnoud \c
 .sp
 The MIT License
 .sp
-Copyright (c) 2009\-2018 \c
+Copyright (c) 2009\-2019 \c
 .MTO "laurent\(atspkdev.net" "Laurent Arnoud" ""

data/test/core_test.rb

@@ -1,9 +1,11 @@
-require File.expand_path('../test_helper', __FILE__)
+# frozen_string_literal: true
+
+require File.expand_path('test_helper', __dir__)
 
 describe ValidateWebsite::Core do
   describe 'invalid options' do
     it 'raise ArgumentError on wrong validation_type' do
-      proc { ValidateWebsite::Core.new({ color: false }, :fail) }
+      _(proc { ValidateWebsite::Core.new({ color: false }, :fail) })
         .must_raise ArgumentError
     end
   end

data/test/crawler_test.rb

@@ -1,4 +1,6 @@
-require File.expand_path('../test_helper', __FILE__)
+# frozen_string_literal: true
+
+require File.expand_path('test_helper', __dir__)
 
 # rubocop:disable Metrics/BlockLength
 describe ValidateWebsite::Crawl do
@@ -21,7 +23,7 @@ describe ValidateWebsite::Crawl do
       _out, _err = capture_io do
         v = ValidateWebsite::Crawl.new(site: TEST_DOMAIN, user_agent: ua)
         v.crawl
-        v.crawler.user_agent.must_equal ua
+        _(v.crawler.user_agent).must_equal ua
       end
     end
 
@@ -31,7 +33,7 @@ describe ValidateWebsite::Crawl do
       _out, _err = capture_io do
         ValidateWebsite::Crawl.new(site: TEST_DOMAIN,
                                    html5_validator_service_url: new)
-        validator.html5_validator_service_url.must_equal new
+        _(validator.html5_validator_service_url).must_equal new
         validator.html5_validator_service_url = original
       end
     end
@@ -43,7 +45,8 @@ describe ValidateWebsite::Crawl do
       _out, _err = capture_io do
         v = ValidateWebsite::Crawl.new(site: TEST_DOMAIN, cookies: cookies)
         v.crawl
-        v.crawler.cookies.cookies_for_host(v.host).must_equal v.default_cookies
+        _(v.crawler.cookies.cookies_for_host(v.host))
+          .must_equal v.default_cookies
       end
     end
   end
@@ -53,37 +56,53 @@ describe ValidateWebsite::Crawl do
       name = 'xhtml1-strict'
       file = File.join('test', 'data', "#{name}.html")
       page = FakePage.new(name,
-                          body: open(file).read,
+                          body: File.open(file).read,
                           content_type: 'text/html')
       @validate_website.site = page.url
       _out, _err = capture_io do
         @validate_website.crawl
       end
-      @validate_website.history_count.must_equal 5
+      _(@validate_website.history_count).must_equal 5
     end
 
     it 'extract link' do
       name = 'html4-strict'
       file = File.join('test', 'data', "#{name}.html")
       page = FakePage.new(name,
-                          body: open(file).read,
+                          body: File.open(file).read,
                           content_type: 'text/html')
       @validate_website.site = page.url
       _out, _err = capture_io do
         @validate_website.crawl
       end
-      @validate_website.history_count.must_equal 98
+      _(@validate_website.history_count).must_equal 98
+    end
+
+    it 'can change validator' do
+      name = 'html5-fail'
+      file = File.join('test', 'data', "#{name}.html")
+      page = FakePage.new(name,
+                          body: File.open(file).read,
+                          content_type: 'text/html')
+      validator_res = File.join('test', 'data', 'validator.nu-failure.json')
+      stub_request(:any, /#{validator.html5_validator_service_url}/)
+        .to_return(body: File.open(validator_res).read)
+      @validate_website.site = page.url
+      _out, _err = capture_io do
+        @validate_website.crawl(html5_validator: :nu, ignore: /Warning/)
+      end
+      _(@validate_website.errors_count).must_equal 1
     end
 
     it 'crawl when URLs are not ascii only' do
       name = 'cozy-community'
       file = File.join('test', 'data', "#{name}.html")
       page = FakePage.new(name,
-                          body: open(file).read,
+                          body: File.open(file).read,
                           content_type: 'text/html')
       validator_res = File.join('test', 'data', 'validator.nu-failure.json')
       stub_request(:any, /#{validator.html5_validator_service_url}/)
-        .to_return(body: open(validator_res).read)
+        .to_return(body: File.open(validator_res).read)
       @validate_website.site = page.url
       _out, _err = capture_io do
         @validate_website.crawl
@@ -119,7 +138,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.history_count.must_equal 5
+        _(@validate_website.history_count).must_equal 5
       end
 
       it 'should extract url with single quote' do
@@ -130,7 +149,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.history_count.must_equal 2
+        _(@validate_website.history_count).must_equal 2
       end
 
       it 'should extract url with double quote' do
@@ -141,7 +160,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.history_count.must_equal 2
+        _(@validate_website.history_count).must_equal 2
       end
 
       it 'should extract url with params' do
@@ -152,7 +171,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.history_count.must_equal 2
+        _(@validate_website.history_count).must_equal 2
       end
 
       it 'should not extract invalid urls' do
@@ -163,7 +182,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.history_count.must_equal 1
+        _(@validate_website.history_count).must_equal 1
       end
     end
 
@@ -182,7 +201,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.errors_count.must_equal 1
+        _(@validate_website.errors_count).must_equal 1
       end
 
       it 'should be invalid with syntax error' do
@@ -193,7 +212,7 @@ describe ValidateWebsite::Crawl do
         _out, _err = capture_io do
           @validate_website.crawl
         end
-        @validate_website.errors_count.must_equal 1
+        _(@validate_website.errors_count).must_equal 1
       end
     end
   end