web-checker 0.4 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d2121dbc5997e578f935a0cc26174e9c714d4d7d0ea57968a3161310290b630
-  data.tar.gz: 8799aef79293dcc714a7abae2d917c9e665367a57e4b61481065966f02874aa4
+  metadata.gz: 5b78dbee67359c44fa8f2bed401f9d308f700c18602babff9177e65d77b1623e
+  data.tar.gz: 517db6e636ebc4d7ff6f136c6fef49549848bba9a99140efbf6392c645b6a178
 SHA512:
-  metadata.gz: 59c2cf78d72952d6989968cfdfb43d6919833398aa662293d16805e418f0a9684d354bc6edf59f8bcbf24c66dc0cf7138e872a3e54abda4bd417a3bed0942799
-  data.tar.gz: 9d280283a9067380d5196e9b87cc5c965fa7cb587f10ed195423df415aeeb887cd7117169ec72da7fde486dc3490df56a2532fbd19141891054d5e7857648131
+  metadata.gz: 354cdfcb90dd17744ccb1e925405ce6b564530f07b5e46414e99727671c2d8dd9afbad4cd4b4ba41b961e3290ad8906782d801bdb7364980efe08a1fd535a46b
+  data.tar.gz: aee34beaad0221d622e52dd95b9002d24d07401fb4f7e3ec2ae08efcf55646585ab8b01a47bff654f316f89b3b69280cbf0d2ab35ce76aa7347c95d912a86cf5

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    web-checker (0.5)
+      addressable (~> 2.8)
+      http (~> 5.0)
+      nokogiri (~> 1.12)
+      path (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.15.4)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    http (5.0.2)
+      addressable (~> 2.8)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      llhttp-ffi (~> 0.4.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    llhttp-ffi (0.4.0)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
+    nokogiri (1.12.4-arm64-darwin)
+      racc (~> 1.4)
+    path (2.0.1)
+    public_suffix (4.0.6)
+    racc (1.5.2)
+    rake (13.0.6)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8)
+
+PLATFORMS
+  arm64-darwin-20
+
+DEPENDENCIES
+  rake (~> 13.0)
+  web-checker!
+
+BUNDLED WITH
+   2.2.27

data/Rakefile

@@ -1,2 +1,9 @@
-require 'rubygems/tasks'
-Gem::Tasks.new
+require 'bundler/gem_tasks'
+Bundler.require
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/*test.rb']
+end
+
+task :default => :test

data/lib/web-checker/version.rb

@@ -1,5 +1,5 @@
 class WebChecker
 
-  VERSION = '0.4'
+  VERSION = '0.5'
 
 end

data/lib/web-checker.rb

@@ -1,26 +1,14 @@
 require 'addressable'
 require 'http'
 require 'nokogiri'
-require 'nokogumbo'
 require 'path'
 
 class WebChecker
 
-  IgnoreErrors = %Q{
-    <table> lacks "summary" attribute
-    <img> lacks "alt" attribute
-    <form> proprietary attribute "novalidate"
-    <input> attribute "type" has invalid value "email"
-    <input> attribute "tabindex" has invalid value "-1"
-    <input> proprietary attribute "border"
-    trimming empty <p>
-    <iframe> proprietary attribute "allowfullscreen"
-  }.split(/\n/).map(&:strip)
-  LinkElementsXPath = '//@href | //@src'
   SchemasDir = Path.new(__FILE__).dirname / 'web-checker' / 'schemas'
-  Schemas = {
-    'feed' => SchemasDir / 'atom.xsd',
-    'urlset' => SchemasDir / 'sitemap.xsd',
+  SchemaNames = {
+    'feed' => 'atom',
+    'urlset' => 'sitemap',
   }
 
   class Error < Exception; end
@@ -29,7 +17,8 @@ class WebChecker
     @site_uri = Addressable::URI.parse(site_uri)
     @site_dir = Path.new(site_dir)
     @schemas = {}
-    @visited = {}
+    @seen = {}
+    @files = []
   end
 
   def check
@@ -41,26 +30,27 @@ class WebChecker
   def check_uri(uri)
     uri = Addressable::URI.parse(uri)
     uri.normalize!
-    return unless local?(uri) && !seen?(uri)
-    # ;;warn "CHECKING: #{uri}"
+    return if seen?(uri)
+    return unless http?(uri)
+    is_local = local?(uri)
+    ;;warn "CHECKING: #{uri}"
     response = HTTP.get(uri)
     # ;;pp(response: response)
-    @visited[uri] = true
+    @seen[uri] = true
     case response.code
     when 200...300
-      body = response.body.to_s
-      # ;;pp(body: body)
-      case (type = response.headers['Content-Type'])
-      when 'text/html'
-        check_html(uri, body)
-      when 'text/css'
-        check_css(uri, body)
-      when 'application/xml', 'text/xml'
-        check_xml(uri, body)
-      when 'image/jpeg', 'image/png', 'image/gif', 'application/javascript'
-        # ignore
-      else
-        ;;warn "skipping unknown resource type: #{uri} (#{type})"
+      if is_local
+        data = response.body.to_s
+        case (type = response.headers['Content-Type'])
+        when 'text/html', 'text/xml', 'application/xml'
+          check_markup(uri, data)
+        when 'text/css'
+          check_css(uri, data)
+        when %r{^image/}, 'application/javascript'
+          # ignore
+        else
+          ;;warn "skipping unknown resource type: #{uri} (#{type})"
+        end
       end
     when 300...400
       redirect_uri = Addressable::URI.parse(response.headers['Location'])
@@ -72,60 +62,32 @@ class WebChecker
     end
   end
 
-  def check_html(uri, html)
-    check_html_tidy(uri, html)
-    check_html_nokogiri(uri, html)
-  end
-
-  def check_html_tidy(uri, html)
-    tmp_file = Path.tmpfile
-    tmp_file.write(html)
-    errors = %x{tidy -utf8 -quiet -errors #{tmp_file} 2>&1}.split("\n")
-    errors = errors.map { |str|
-      # line 82 column 1 - Warning: <table> lacks "summary" attribute
-      str =~ /^line (\d+) column (\d+) - (.*?): (.*)$/ or raise "Can't parse error: #{str.inspect}"
-      {
-        msg: str,
-        line: $1.to_i,
-        column: $2.to_i,
-        type: $3.downcase.to_sym,
-        error: $4.strip,
-      }
-    }.reject { |e|
-      IgnoreErrors.include?(e[:error])
-    }
-    unless errors.empty?
-      warn "#{uri} has invalid HTML"
-      show_errors(errors)
-      raise Error, "HTML parsing failed (via Tidy)"
+  def check_markup(uri, data)
+# ;;warn "validating markup: #{uri}"
+    doc = case data
+    when /^<\?xml/i
+      Nokogiri::XML(data) { |c| c.strict }
+    when /^<!DOCTYPE html>/i
+      Nokogiri::HTML5(data, max_errors: -1)
+    else
+      Nokogiri::HTML4(data) { |c| c.strict }
     end
-  end
-
-  def check_html_nokogiri(uri, html)
-    doc_class = (html =~ /<!DOCTYPE html>/i) ? Nokogiri::HTML5 : Nokogiri::HTML
-    doc = doc_class.parse(html) { |config| config.strict }
     unless doc.errors.empty?
       show_errors(doc.errors)
-      raise Error, "HTML parsing failed (via Nokogiri)"
+      raise Error, "markup parsing failed"
     end
-    doc.xpath(LinkElementsXPath).each { |e| check_uri(uri + e.value) }
-  end
-
-  def check_xml(uri, xml)
-    xml_doc = Nokogiri::XML::Document.parse(xml) { |config| config.strict }
-    unless xml_doc.errors.empty?
-      show_errors(xml_doc.errors)
-      raise Error, "XML parsing failed"
+    if (schema_name = SchemaNames[doc.root.name])
+      schema_file = (SchemasDir / schema_name).add_extension('.xsd')
+      schema = (@schemas[schema_file] ||= Nokogiri::XML::Schema(schema_file.open))
+      validation_errors = schema.validate(doc)
+      unless validation_errors.empty?
+        show_errors(validation_errors)
+        raise Error, "schema validation failed"
+      end
     end
-    root_name = xml_doc.root.name
-    schema_file = Schemas[root_name] or raise Error, "Unknown schema: #{root_name.inspect}"
-    schema = (@schemas[schema_file] ||= Nokogiri::XML::Schema(schema_file.open))
-    validation_errors = schema.validate(xml_doc)
-    unless validation_errors.empty?
-      show_errors(validation_errors)
-      raise Error, "XML validation failed"
+    doc.xpath('//@href | //@src').each do |elem|
+      check_uri(uri + elem.value)
     end
-    xml_doc.xpath(LinkElementsXPath).each { |e| check_uri(uri + e.value) }
   end
 
   def show_errors(errors)
@@ -140,13 +102,17 @@ class WebChecker
     end
   end
 
+  def http?(uri)
+    !uri.scheme || %w[http https].include?(uri.scheme)
+  end
+
   def local?(uri)
     (!uri.scheme && !uri.host) ||
       (uri.scheme == @site_uri.scheme && uri.host == @site_uri.host && uri.port == @site_uri.port)
   end
 
   def seen?(uri)
-    @visited[uri]
+    @seen[uri]
   end
 
   def report

data/web-checker.gemspec

@@ -18,12 +18,10 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_path  = 'lib'
 
-  s.add_dependency 'addressable', '~> 2'
-  s.add_dependency 'http', '~> 4'
-  s.add_dependency 'nokogiri', '~> 1'
-  s.add_dependency 'nokogumbo', '~> 2'
-  s.add_dependency 'path', '~> 2'
+  s.add_dependency 'addressable', '~> 2.8'
+  s.add_dependency 'http', '~> 5.0'
+  s.add_dependency 'nokogiri', '~> 1.12'
+  s.add_dependency 'path', '~> 2.0'
 
-  s.add_development_dependency 'rake', '~> 12'
-  s.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  s.add_development_dependency 'rake', '~> 13.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web-checker
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
 platform: ruby
 authors:
 - John Labovitz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-29 00:00:00.000000000 Z
+date: 2021-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,98 +16,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
-- !ruby/object:Gem::Dependency
-  name: nokogumbo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: path
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '12'
-- !ruby/object:Gem::Dependency
-  name: rubygems-tasks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '13.0'
 description: "\n    WebChecker checks static websites for consistency.\n  "
 email: johnl@johnlabovitz.com
 executables:
@@ -116,6 +88,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- Gemfile
+- Gemfile.lock
 - Rakefile
 - bin/web-checker
 - lib/web-checker.rb
@@ -127,7 +101,7 @@ homepage: http://github.com/jslabovitz/web-checker
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -142,9 +116,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.2.27
+signing_key:
 specification_version: 4
 summary: Check static websites for consistency.
 test_files: []