validate-website 1.1.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 134cbdd3da2da6847c525ffe5a2ee68f1a380ae2
-  data.tar.gz: 678ff514b9f6f368bbb78e93a8dc42f38a35a803
+  metadata.gz: fd93ea8bbe188ac4cb33752c93a971f75106971e
+  data.tar.gz: 9322f70f9cd7433c35d201f0d042117ff2974147
 SHA512:
-  metadata.gz: dd90c5dec7d0c80ea9b94abcb1a38a425ee59be32c73fe9fc97620a5a00ee4c9cf2dae52aa099509c8573f2dfca117377ed15373bce2d93ed7c25a43e6ed067e
-  data.tar.gz: 4d24ac9b1dccd744a3b7bd7ebbb18b946c05c4043a0c76898eafc543892e886715e2d99670ea91f3b9d0c3203425cf0de420445d041360c44fb67a934c6165c2
+  metadata.gz: e1d587ebf07f0f3f93ee4f7ebfd96112919c435eeddda17874c76423ee0e1500865a0afe0f5112c459cd900023445e818d347380800c306c0d398ed9369de710
+  data.tar.gz: 818c8b7f437a4afb0482914fd799e0e0701013ee9b772db56abe3e3aa2259f08cba9444fff65df278b10a8e4144aa08021e38515419ccfebe832c717d22fa63b

data/Rakefile

@@ -1,13 +1,8 @@
-require 'rdoc/task'
 require 'rake/testtask'
+require 'rubocop/rake_task'
 
 task default: [:test]
 
-RDoc::Task.new do |rd|
-  rd.main = 'README.rdoc'
-  rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb')
-end
-
 # install asciidoc libxml2-utils xmlto docbook-xsl docbook-xml
 desc 'Update manpage from asciidoc file'
 task :manpage do
@@ -15,6 +10,11 @@ task :manpage do
 end
 
 Rake::TestTask.new do |t|
-  t.pattern = 'spec/*_spec.rb'
+  t.pattern = 'test/**/*_test.rb'
 end
 task spec: :test
+
+desc 'Execute rubocop'
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['-D'] # display cop name
+end

data/lib/validate_website/core.rb

@@ -7,6 +7,7 @@ require 'validate_website/validator'
 require 'validate_website/colorful_messages'
 
 require 'spidr'
+require 'crass'
 
 # Base module ValidateWebsite
 module ValidateWebsite
@@ -62,20 +63,25 @@ module ValidateWebsite
       end
     end
 
-    # Extract urls from CSS page
-    #
-    # @param [Spidr::Page] an Spidr::Page object
-    # @return [Array] Lists of urls
-    #
-    def self.extract_urls_from_css(page)
-      page.body.scan(%r{url\((['".\/\w-]+)\)}).reduce(Set[]) do |result, url|
-        url = url.first.gsub("'", '').gsub('"', '')
-        abs = page.to_absolute(url)
-        result << abs.to_s
-      end
+    private
+
+    def check_css_syntax(page)
+      nodes = Crass::Parser.parse_stylesheet(page.body)
+      return unless any_css_errors?(nodes)
+      handle_validation_error(page.url)
     end
 
-    private
+    def any_css_errors?(nodes)
+      nodes.any? do |node|
+        if node[:children]
+          any_css_errors? node.delete(:children)
+        elsif node[:tokens]
+          any_css_errors? node.delete(:tokens)
+        else
+          node[:node] == :error || node[:error] == true
+        end
+      end
+    end
 
     def print_status_line(total, failures, not_founds, errors)
       puts "\n\n"
@@ -102,16 +108,20 @@ module ValidateWebsite
       if validator.valid?
         print color(:success, '.', options[:color]) # rspec style
       else
-        handle_validation_error(validator, url)
+        handle_html_validation_error(validator, url)
       end
     end
 
-    def handle_validation_error(validator, url)
+    def handle_html_validation_error(validator, url)
+      handle_validation_error(url)
+      return unless options[:verbose]
+      puts color(:error, validator.errors.join(', '), options[:color])
+    end
+
+    def handle_validation_error(url)
       @errors_count += 1
       puts "\n"
       puts color(:error, "* #{url}", options[:color])
-      return unless options[:verbose]
-      puts color(:error, validator.errors.join(', '), options[:color])
     end
   end
 end

data/lib/validate_website/crawl.rb

@@ -1,4 +1,5 @@
 require 'validate_website/core'
+require 'validate_website/utils'
 
 module ValidateWebsite
   # Class for http website validation
@@ -44,13 +45,14 @@ module ValidateWebsite
         crawler.cookies[@host] = default_cookies if options[:cookies]
         on_every_css_page(crawler)
         on_every_html_page(crawler)
-        on_every_failed_url(crawler)
+        on_every_failed_url(crawler) if options[:not_found]
       end
     end
 
     def on_every_css_page(crawler)
       crawler.every_css_page do |page|
-        ValidateWebsite::Core.extract_urls_from_css(page).each do |u|
+        check_css_syntax(page) if options[:css_syntax]
+        ValidateWebsite::Utils.extract_urls_from_css(page).each do |u|
           crawler.enqueue(u)
         end
       end
@@ -69,7 +71,6 @@ module ValidateWebsite
     end
 
     def on_every_failed_url(crawler)
-      return unless options[:not_found]
       crawler.every_failed_url do |url|
         not_found_error(url)
       end

data/lib/validate_website/option_parser.rb

@@ -12,6 +12,7 @@ module ValidateWebsite
       exclude: nil,
       user_agent: nil,
       markup: true,
+      css_syntax: false,
       # crawler: log not found url (404 status code)
       # static: log not found url (not on filesystem, `pwd` considered
       # as root " / ")
@@ -39,6 +40,7 @@ module ValidateWebsite
     def self.default_args
       Slop.parse do |o|
         yield o if block_given?
+        markup_syntax(o)
         boolean_options(o)
         o.regexp('-i', '--ignore',
                  'Validation errors to ignore (ex: "valign|autocorrect")')
@@ -48,10 +50,16 @@ module ValidateWebsite
       end
     end
 
-    def self.boolean_options(o)
+    def self.markup_syntax(o)
       o.bool('-m', '--markup',
              "Markup validation (default: #{DEFAULT_OPTIONS[:markup]})",
              default: DEFAULT_OPTIONS[:markup])
+      o.bool('--css-syntax',
+             "Css validation (default: #{DEFAULT_OPTIONS[:css_syntax]})",
+             default: DEFAULT_OPTIONS[:css_syntax])
+    end
+
+    def self.boolean_options(o)
       o.bool('-n', '--not-found',
              "Log not found url (default: #{DEFAULT_OPTIONS[:not_found]})",
              default: DEFAULT_OPTIONS[:not_found])
@@ -94,7 +102,7 @@ module ValidateWebsite
         o.string('-s', '--site',
                  "Website to crawl (default: #{DEFAULT_OPTIONS[:site]})",
                  default: DEFAULT_OPTIONS[:site])
-        o.regexp('-p', '--pattern',
+        o.string('-p', '--pattern',
                  "Filename pattern (default: #{DEFAULT_OPTIONS[:pattern]})",
                  default: DEFAULT_OPTIONS[:pattern])
       end

data/lib/validate_website/static.rb

@@ -1,4 +1,5 @@
 require 'validate_website/core'
+require 'validate_website/utils'
 
 module ValidateWebsite
   # Class for validation Static website
@@ -25,15 +26,17 @@ module ValidateWebsite
 
     private
 
-    def generate_static_page(f)
-      response = self.class.fake_httpresponse(open(f).read)
-      Spidr::Page.new(URI.join(@site, URI.encode(f)), response)
+    def check_static_file(f)
+      page = StaticLink.new(f, @site).page
+      check_page(f, page)
+      check_css_syntax(page) if page.css? && options[:css_syntax]
     end
 
-    def check_static_file(f)
-      page = generate_static_page(f)
-      validate(page.doc, page.body, f, @options[:ignore]) if @options[:markup]
-      check_static_not_found(page.links) if @options[:not_found]
+    def check_page(f, page)
+      if page.html? && options[:markup]
+        validate(page.doc, page.body, f, options[:ignore])
+      end
+      check_static_not_found(page.links) if options[:not_found]
     end
 
     StaticLink = Struct.new(:link, :site) do
@@ -47,12 +50,34 @@ module ValidateWebsite
         URI.parse(site).host == link_uri.host
       end
 
+      def content_types
+        if css?
+          ['text/css']
+        else
+          CONTENT_TYPES
+        end
+      end
+
+      def body
+        if File.exist?(link)
+          open(link).read
+        else
+          open(file_path).read
+        end
+      end
+
+      def response
+        @response ||= ValidateWebsite::Static.fake_httpresponse(
+          body,
+          content_types)
+      end
+
+      def page
+        @page ||= Spidr::Page.new(link_uri, response)
+      end
+
       def extract_urls_from_fake_css_response
-        response = ValidateWebsite::Static.fake_httpresponse(
-          open(file_path).read,
-          ['text/css'])
-        css_page = Spidr::Page.new(link_uri, response)
-        ValidateWebsite::Core.extract_urls_from_css(css_page)
+        ValidateWebsite::Utils.extract_urls_from_css(page)
       end
 
       def file_path
@@ -65,6 +90,10 @@ module ValidateWebsite
         @extname ||= File.extname(file_path)
       end
 
+      def css?
+        extname == '.css'
+      end
+
       def check?
         !link.include?('#') && in_static_domain?
       end
@@ -78,7 +107,7 @@ module ValidateWebsite
         next unless static_link.check?
         not_found_error(static_link.file_path) &&
           next unless File.exist?(static_link.file_path)
-        next unless static_link.extname == '.css'
+        next unless static_link.css?
         check_static_not_found static_link.extract_urls_from_fake_css_response
       end
     end

data/lib/validate_website/utils.rb

@@ -0,0 +1,48 @@
+# Base module ValidateWebsite
+module ValidateWebsite
+  # Utils class for CSS helpers
+  class Utils
+    # Extract urls from CSS page
+    #
+    # @param [Spidr::Page] a Spidr::Page object
+    # @return [Set] Lists of urls
+    #
+    def self.extract_urls_from_css(page)
+      return Set[] unless page
+      return Set[] if page.body.nil?
+      nodes = Crass::Parser.parse_stylesheet(page.body)
+      extract_urls_from_nodes nodes, page
+    end
+
+    # Return urls as absolute from Crass nodes
+    #
+    # @param [Hash] node from Crass
+    # @param [Spidr::Page] a Spidr::Page object
+    # @return [Set] list of obsolute urls
+    def self.urls_to_absolute(node, page)
+      if node[:node] == :function && node[:name] == 'url' || node[:node] == :url
+        Array(node[:value]).map do |v|
+          url = v.is_a?(String) ? v : v[:value]
+          page.to_absolute(url).to_s
+        end
+      else
+        Set.new
+      end
+    end
+
+    # Extract urls from Crass nodes
+    # @param [Array] Array of nodes from Crass
+    # @param [Spidr::Page] a Spidr::Page object
+    # @param [Set] memo for recursivity
+    # @return [Set] list of urls
+    def self.extract_urls_from_nodes(nodes, page, memo = Set[])
+      nodes.each_with_object(memo) do |node, result|
+        result.merge urls_to_absolute(node, page)
+        if node[:children]
+          extract_urls_from_nodes node.delete(:children), page, result
+        end
+        result
+      end
+    end
+  end
+end

data/lib/validate_website/version.rb

@@ -1,3 +1,4 @@
+# Version file for ValidateWebsite
 module ValidateWebsite
-  VERSION = '1.1.0'.freeze
+  VERSION = '1.5.0'.freeze
 end

data/lib/validate_website.rb

@@ -1,3 +1,2 @@
-# encoding: utf-8
 require 'validate_website/core'
 require 'validate_website/version'

data/man/man1/validate-website-static.1

@@ -2,12 +2,12 @@
 .\"     Title: validate-website-static
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 05/17/2015
+.\"      Date: 07/21/2015
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE\-S" "1" "05/17/2015" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE\-S" "1" "07/21/2015" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -59,6 +59,11 @@ Ignore certain validation errors (ex:
 Markup validation (Default: true)
 .RE
 .PP
+\fB\-\-css\-syntax\fR
+.RS 4
+Css validation (Default: false)
+.RE
+.PP
 \fB\-n\fR, \fB\-\-not\-found\fR
 .RS 4
 Log files not on filesystem, pwd considered as root \(Fo / \(Fc (Default: false)

data/man/man1/validate-website.1

@@ -2,12 +2,12 @@
 .\"     Title: validate-website
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 05/17/2015
+.\"      Date: 07/21/2015
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE" "1" "05/17/2015" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE" "1" "07/21/2015" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -70,6 +70,11 @@ Set defaults cookies
 Markup validation (Default: true)
 .RE
 .PP
+\fB\-\-css\-syntax\fR
+.RS 4
+Css validation (Default: false)
+.RE
+.PP
 \fB\-n\fR, \fB\-\-not\-found\fR
 .RS 4
 Log not found url (Default: false)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: validate-website
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Laurent Arnoud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-07 00:00:00.000000000 Z
+date: 2015-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: crass
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: paint
   requirement: !ruby/object:Gem::Requirement
@@ -246,27 +260,11 @@ files:
 - lib/validate_website/option_parser.rb
 - lib/validate_website/runner.rb
 - lib/validate_website/static.rb
+- lib/validate_website/utils.rb
 - lib/validate_website/validator.rb
 - lib/validate_website/version.rb
 - man/man1/validate-website-static.1
 - man/man1/validate-website.1
-- spec/core_spec.rb
-- spec/crawler_spec.rb
-- spec/data/assets/application-92f19110a9d47a56d2ebe744e15af301.css
-- spec/data/html4-strict.html
-- spec/data/html5-linuxfr.html
-- spec/data/html5.html
-- spec/data/news/ryzom-naissance-du-projet-libre-ryzom-forge.md
-- spec/data/validator.nu-excessive.html
-- spec/data/validator.nu-failure.html
-- spec/data/validator.nu-success.html
-- spec/data/w3.org-xhtml1-strict-errors.html
-- spec/data/xhtml1-strict.html
-- spec/example/ruby smalltalk/blockcamp-paris-le-28-novembre.html
-- spec/spec_helper.rb
-- spec/static_spec.rb
-- spec/validator_spec.rb
-- spec/webmock_helper.rb
 homepage: http://github.com/spk/validate-website
 licenses:
 - MIT
@@ -294,9 +292,5 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Web crawler for checking the validity of your documents
-test_files:
-- spec/static_spec.rb
-- spec/core_spec.rb
-- spec/crawler_spec.rb
-- spec/validator_spec.rb
+test_files: []
 has_rdoc: