nanoc 3.4.3 → 3.5.0b1

data/lib/nanoc/data_sources.rb

@@ -5,9 +5,11 @@ module Nanoc::DataSources
   autoload 'Filesystem',         'nanoc/data_sources/filesystem'
   autoload 'FilesystemUnified',  'nanoc/data_sources/filesystem_unified'
   autoload 'FilesystemVerbose',  'nanoc/data_sources/filesystem_verbose'
+  autoload 'Static',             'nanoc/data_sources/static'
 
   Nanoc::DataSource.register '::Nanoc::DataSources::FilesystemVerbose',  :filesystem_verbose
   Nanoc::DataSource.register '::Nanoc::DataSources::FilesystemUnified',  :filesystem_unified
+  Nanoc::DataSource.register '::Nanoc::DataSources::Static',             :static
 
   # Deprecated; fetch data from online data sources manually instead
   # TODO [in nanoc 4.0] remove me

data/lib/nanoc/data_sources/filesystem_unified.rb

@@ -85,7 +85,7 @@ module Nanoc::DataSources
       # Write item
       FileUtils.mkdir_p(parent_path)
       File.open(path, 'w') do |io|
-        meta = attributes.stringify_keys
+        meta = attributes.stringify_keys_recursively
         unless meta == {}
           io.write(YAML.dump(meta).strip + "\n")
           io.write("---\n\n")

data/lib/nanoc/data_sources/filesystem_verbose.rb

@@ -63,7 +63,7 @@ module Nanoc::DataSources
 
       # Create files
       FileUtils.mkdir_p(dir_path)
-      File.open(meta_filename,    'w') { |io| io.write(YAML.dump(attributes.stringify_keys)) }
+      File.open(meta_filename,    'w') { |io| io.write(YAML.dump(attributes.stringify_keys_recursively)) }
       File.open(content_filename, 'w') { |io| io.write(content) }
     end
 

data/lib/nanoc/data_sources/static.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+
+module Nanoc::DataSources
+
+  # The static data source provides items from a single directory. Unlike the
+  # filesystem data sources, static provides no additional item metadata. In
+  # addition, all items are treated as 'binary', regardless of their extension
+  # or content. As such, it is most useful for simple assets, not for normal
+  # content.
+  #
+  # The identifier for static items is the full item path. For example, if your
+  # static data source item_root is `static`, an item named `foo.css` would have
+  # the identifier `/static/foo.css/`. Note that, unlike the filesystem data
+  # sources, `foo/index.html` and `foo.yaml` receive no special treatment. They
+  # are simple static items, just like `foo.css`.
+  #
+  # The default data source directory is `static/`, but this can be overridden
+  # in the data source configuration:
+  #
+  #    data_sources:
+  #      - type:   static
+  #        prefix: assets
+  #
+  # Unless the `hide_items` configuration attribute is false, items from static
+  # data sources will have the :is_hidden attribute set by default, which will
+  # exclude them from the Blogging helper's atom feed generator, among other
+  # things.
+  class Static < Nanoc::DataSource
+    identifier :static
+
+    def items
+      # Get prefix
+      prefix = config[:prefix] || 'static'
+
+      # Get all files under prefix dir
+      filenames = Dir[prefix + '/**/*'].select { |f| File.file?(f) }
+
+      # Convert filenames to items
+      filenames.map do |filename|
+        attributes = {
+          :extension => File.extname(filename)[1..-1],
+          :filename  => filename,
+        }
+        attributes[:is_hidden] = true unless config[:hide_items] == false
+        identifier = filename[(prefix.length+1)..-1] + '/'
+
+        mtime      = File.mtime(filename)
+        checksum   = Pathname.new(filename).checksum
+
+        Nanoc::Item.new(
+          filename,
+          attributes,
+          identifier,
+          :binary => true, :mtime => mtime, :checksum => checksum
+        )
+      end
+    end
+
+  end
+end

data/lib/nanoc/extra.rb

@@ -3,7 +3,9 @@
 module Nanoc::Extra
 
   autoload 'AutoCompiler',      'nanoc/extra/auto_compiler'
+  autoload 'Checking',          'nanoc/extra/checking'
   autoload 'CHiCk',             'nanoc/extra/chick'
+  autoload 'LinkCollector',     'nanoc/extra/link_collector.rb'
   autoload 'Pruner',            'nanoc/extra/pruner'
   autoload 'Validators',        'nanoc/extra/validators'
 

data/lib/nanoc/extra/checking.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+
+module Nanoc::Extra
+
+  module Checking
+
+    autoload 'Check',  'nanoc/extra/checking/check'
+    autoload 'DSL',    'nanoc/extra/checking/dsl'
+    autoload 'Runner', 'nanoc/extra/checking/runner.rb'
+    autoload 'Issue',  'nanoc/extra/checking/issue'
+
+  end
+
+end
+
+require 'nanoc/extra/checking/checks'

data/lib/nanoc/extra/checking/check.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+module Nanoc::Extra::Checking
+
+  class Check
+
+    extend Nanoc::PluginRegistry::PluginMethods
+
+    attr_reader :site
+    attr_reader :issues
+
+    def initialize(site)
+      @site   = site
+      @issues = Set.new
+    end
+
+    def run
+      raise NotImplementedError.new("Nanoc::Extra::Checking::Check subclasses must implement #run")
+    end
+
+    def add_issue(desc, params={})
+      subject  = params.fetch(:subject, nil)
+
+      @issues << Issue.new(desc, subject, self.class)
+    end
+
+    def output_filenames
+      Dir[@site.config[:output_dir] + '/**/*'].select{ |f| File.file?(f) }
+    end
+
+  end
+
+end

data/lib/nanoc/extra/checking/checks.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+module Nanoc::Extra::Checking::Checks
+
+  autoload 'CSS',           'nanoc/extra/checking/checks/css'
+  autoload 'ExternalLinks', 'nanoc/extra/checking/checks/external_links'
+  autoload 'HTML',          'nanoc/extra/checking/checks/html'
+  autoload 'InternalLinks', 'nanoc/extra/checking/checks/internal_links'
+  autoload 'Stale',         'nanoc/extra/checking/checks/stale'
+
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::CSS',           :css
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::ExternalLinks', :external_links
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::ExternalLinks', :elinks
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::HTML',          :html
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::InternalLinks', :internal_links
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::InternalLinks', :ilinks
+  Nanoc::Extra::Checking::Check.register '::Nanoc::Extra::Checking::Checks::Stale',         :stale
+
+end

data/lib/nanoc/extra/checking/checks/css.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8   
+
+module ::Nanoc::Extra::Checking::Checks
+
+  class CSS < ::Nanoc::Extra::Checking::Check
+
+    identifier :css
+
+    def run
+      require 'w3c_validators'
+
+      Dir[site.config[:output_dir] + '/**/*.css'].each do |filename|
+        results = ::W3CValidators::CSSValidator.new.validate_file(filename)
+        results.errors.each do |e|
+          desc = e.message.gsub(%r{\s+}, ' ').strip
+          self.add_issue(desc, :subject => filename)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/nanoc/extra/checking/checks/external_links.rb

@@ -0,0 +1,149 @@
+# encoding: utf-8
+
+require 'net/http'
+require 'net/https'
+require 'nokogiri'
+require 'timeout'
+require 'uri'
+
+module ::Nanoc::Extra::Checking::Checks
+
+  # A validator that verifies that all external links point to a location that exists.
+  class ExternalLinks < ::Nanoc::Extra::Checking::Check
+
+    identifiers :external_links, :elinks
+
+    def run
+      # Find all broken external hrefs
+      # TODO de-duplicate this (duplicated in internal links check)
+      filenames = self.output_filenames.select { |f| File.extname(f) == '.html' }
+      hrefs_with_filenames = ::Nanoc::Extra::LinkCollector.new(filenames, :external).filenames_per_href
+      results = self.select_invalid(hrefs_with_filenames.keys)
+
+      # Report them
+      results.each do |res|
+        filenames = hrefs_with_filenames[res.href]
+        filenames.each do |filename|
+          self.add_issue(
+            "reference to #{res.href}: #{res.explanation}",
+            :subject => filename)
+        end
+      end
+    end
+
+    class Result
+
+      attr_reader :href
+      attr_reader :explanation
+
+      def initialize(href, explanation)
+        @href        = href
+        @explanation = explanation
+      end
+
+    end
+
+    class ArrayEnumerator
+
+      def initialize(array)
+        @array = array
+        @index = 0
+        @mutex = Mutex.new
+      end
+
+      def next
+        @mutex.synchronize do
+          @index += 1
+          return @array[@index-1]
+        end
+      end
+
+    end
+
+    def select_invalid(hrefs)
+      enum = ArrayEnumerator.new(hrefs.sort)
+      mutex = Mutex.new
+      invalid = Set.new
+
+      threads = []
+      10.times do
+        threads << Thread.new do
+          loop do
+            href = enum.next
+            break if href.nil?
+            res = self.validate(href)
+            if res
+              mutex.synchronize do
+                invalid << res
+              end
+            end
+          end
+        end
+      end
+      threads.each { |t| t.join }
+
+      invalid
+    end
+
+    def validate(href)
+      # Parse
+      url = nil
+      begin
+        url = URI.parse(href)
+      rescue URI::InvalidURIError
+        return Result.new(href, 'invalid URI')
+      end
+
+      # Skip non-HTTP URLs
+      return nil if url.scheme !~ /^https?$/
+
+      # Get status
+      res = nil
+      5.times do |i|
+        begin
+          Timeout::timeout(10) do
+            res = request_url_once(url)
+          end
+        rescue => e
+          return Result.new(href, e.message)
+        end
+
+        if res.code =~ /^3..$/
+          if i == 4
+            return Result.new(href, 'too many redirects')
+          end
+
+          # Find proper location
+          location = res['Location']
+          if location !~ /^https?:\/\//
+            base_url = url.dup
+            base_url.path = (location =~ /^\// ? '' : '/')
+            base_url.query = nil
+            base_url.fragment = nil
+            location = base_url.to_s + location
+          end
+
+          url = URI.parse(location)
+        elsif res.code == '200'
+          return nil
+        else
+          return Result.new(href, res.code)
+        end
+      end
+      raise 'should not have gotten here'
+    end
+
+    def request_url_once(url)
+      path = (url.path.nil? || url.path.empty? ? '/' : url.path)
+      req = Net::HTTP::Head.new(path)
+      http = Net::HTTP.new(url.host, url.port)
+      if url.instance_of? URI::HTTPS
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      res = http.request(req)
+    end
+
+  end
+
+end

data/lib/nanoc/extra/checking/checks/html.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8   
+
+module ::Nanoc::Extra::Checking::Checks
+
+  class HTML < ::Nanoc::Extra::Checking::Check
+
+    identifier :html
+
+    def run
+      require 'w3c_validators'
+
+      Dir[site.config[:output_dir] + '/**/*.{htm,html}'].each do |filename|
+        results = ::W3CValidators::MarkupValidator.new.validate_file(filename)
+        results.errors.each do |e|
+          desc = e.message.gsub(%r{\s+}, ' ').strip
+          self.add_issue(desc, :subject => filename)
+        end
+      end
+    end
+
+  end
+
+end
+

data/lib/nanoc/extra/checking/checks/internal_links.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+module Nanoc::Extra::Checking::Checks
+
+  # A check that verifies that all internal links point to a location that exists.
+  class InternalLinks < ::Nanoc::Extra::Checking::Check
+
+    # Starts the validator. The results will be printed to stdout.
+    #
+    # @return [void]
+    def run
+      # TODO de-duplicate this (duplicated in external links check)
+      filenames = self.output_filenames.select { |f| File.extname(f) == '.html' }
+      hrefs_with_filenames = ::Nanoc::Extra::LinkCollector.new(filenames, :internal).filenames_per_href
+      hrefs_with_filenames.each_pair do |href, filenames|
+        filenames.each do |filename|
+          unless valid?(href, filename)
+          self.add_issue(
+            "reference to #{href}",
+            :subject  => filename)
+          end
+        end
+      end
+    end
+
+  protected
+
+    def valid?(href, origin)
+      # Skip hrefs that point to self
+      # FIXME this is ugly and won’t always be correct
+      return true if href == '.'
+
+      # Remove target
+      path = href.sub(/#.*$/, '')
+      return true if path.empty?
+
+      # Make absolute
+      if path[0, 1] == '/'
+        path = @site.config[:output_dir] + path
+      else
+        path = ::File.expand_path(path, ::File.dirname(origin))
+      end
+
+      # Check whether file exists
+      return true if File.file?(path)
+
+      # Check whether directory with index file exists
+      return true if File.directory?(path) && @site.config[:index_filenames].any? { |fn| File.file?(File.join(path, fn)) }
+
+      # Nope :(
+      return false
+    end
+
+  end
+
+end
+

data/lib/nanoc/extra/checking/checks/stale.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+module Nanoc::Extra::Checking::Checks
+
+  class Stale < ::Nanoc::Extra::Checking::Check
+
+    def run
+      require 'set'
+      item_rep_paths = Set.new(@site.items.collect { |i| i.reps }.flatten.collect { |r| r.raw_path })
+      self.output_filenames.each do |f|
+        if !item_rep_paths.include?(f)
+          self.add_issue(
+            "file without matching item",
+            :subject  => f)
+        end
+      end
+    end
+
+  end
+
+end
+
+