middleman-core 3.3.12 → 4.4.2

data/lib/middleman-core/util/binary.rb

@@ -0,0 +1,78 @@
+# Template and Mime detection
+require 'tilt'
+require 'rack/mime'
+
+require 'middleman-core/contracts'
+
+module Middleman
+  module Util
+    include Contracts
+
+    module_function
+
+    # Whether the source file is binary.
+    #
+    # @param [String] filename The file to check.
+    # @return [Boolean]
+    Contract Or[String, Pathname] => Bool
+    def binary?(filename)
+      @binary_cache ||= {}
+
+      return @binary_cache[filename] if @binary_cache.key?(filename)
+
+      @binary_cache[filename] = begin
+        path = Pathname(filename)
+        ext = path.extname
+
+        # We hardcode detecting of gzipped SVG files
+        if ext == '.svgz'
+          true
+        elsif ::Tilt.registered?(ext.sub('.', ''))
+          false
+        else
+          dot_ext = ext.to_s[0] == '.' ? ext.dup : ".#{ext}"
+
+          if mime = ::Rack::Mime.mime_type(dot_ext, nil)
+            !nonbinary_mime?(mime)
+          else
+            file_contents_include_binary_bytes?(path.to_s)
+          end
+        end
+      end
+    end
+
+    # Is mime type known to be non-binary?
+    #
+    # @param [String] mime The mimetype to check.
+    # @return [Boolean]
+    Contract String => Bool
+    def nonbinary_mime?(mime)
+      if mime.start_with?('text/')
+        true
+      elsif mime.include?('xml') && !mime.include?('officedocument')
+        true
+      elsif mime.include?('json')
+        true
+      elsif mime.include?('javascript')
+        true
+      else
+        false
+      end
+    end
+
+    # Read a few bytes from the file and see if they are binary.
+    #
+    # @param [String] filename The file to check.
+    # @return [Boolean]
+    Contract String => Bool
+    def file_contents_include_binary_bytes?(filename)
+      binary_bytes = [0, 1, 2, 3, 4, 5, 6, 11, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31]
+      s = ::File.read(filename, 4096) || ''
+      s.each_byte do |c|
+        return true if binary_bytes.include?(c)
+      end
+
+      false
+    end
+  end
+end

data/lib/middleman-core/util/data.rb

@@ -0,0 +1,187 @@
+require 'yaml'
+require 'json'
+require 'toml'
+require 'pathname'
+require 'backports/2.1.0/array/to_h'
+require 'hashie'
+require 'memoist'
+
+require 'middleman-core/util/binary'
+require 'middleman-core/contracts'
+
+module Middleman
+  module Util
+    include Contracts
+
+    module_function
+
+    class EnhancedHash < ::Hashie::Mash
+      # include ::Hashie::Extensions::MergeInitializer
+      # include ::Hashie::Extensions::MethodReader
+      # include ::Hashie::Extensions::IndifferentAccess
+    end
+
+    # Recursively convert a normal Hash into a EnhancedHash
+    #
+    # @private
+    # @param [Hash] data Normal hash
+    # @return [Hash]
+    Contract Any => Maybe[Or[Array, EnhancedHash]]
+    def recursively_enhance(obj)
+      if obj.is_a? ::Array
+        obj.map { |e| recursively_enhance(e) }
+      elsif obj.is_a? ::Hash
+        EnhancedHash.new(obj)
+      else
+        obj
+      end
+    end
+
+    module Data
+      extend Memoist
+      include Contracts
+
+      module_function
+
+      # Get the frontmatter and plain content from a file
+      # @param [String] path
+      # @return [Array<Hash, String>]
+      Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]]
+      def parse(file, frontmatter_delims, known_type=nil)
+        full_path = file[:full_path]
+        return [{}, nil] if ::Middleman::Util.binary?(full_path) || file[:types].include?(:binary)
+
+        # Avoid weird race condition when a file is renamed
+        begin
+          content = file.read
+        rescue EOFError, IOError, ::Errno::ENOENT
+          return [{}, nil]
+        end
+
+        match = build_regex(frontmatter_delims).match(content) || {}
+
+        unless match[:frontmatter]
+          case known_type
+          when :yaml
+            return [parse_yaml(content, full_path), nil]
+          when :json
+            return [parse_json(content, full_path), nil]
+          when :toml
+            return [parse_toml(content, full_path), nil]
+          end
+        end
+
+        case [match[:start], match[:stop]]
+        when *frontmatter_delims[:yaml]
+          [
+            parse_yaml(match[:frontmatter], full_path),
+            match[:additional_content]
+          ]
+        when *frontmatter_delims[:json]
+          [
+            parse_json("{#{match[:frontmatter]}}", full_path),
+            match[:additional_content]
+          ]
+        when *frontmatter_delims[:toml]
+          [
+            parse_toml(match[:frontmatter], full_path),
+            match[:additional_content]
+          ]
+        else
+          [
+            {},
+            content
+          ]
+        end
+      end
+
+      def build_regex(frontmatter_delims)
+        start_delims, stop_delims = frontmatter_delims
+                                    .values
+                                    .flatten(1)
+                                    .transpose
+                                    .map(&::Regexp.method(:union))
+
+        /
+          \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)?
+          (?<start>#{start_delims})[ ]*\r?\n
+          (?<frontmatter>.*?)[ ]*\r?\n?
+          ^(?<stop>#{stop_delims})[ ]*\r?\n?
+          \r?\n?
+          (?<additional_content>.*)
+        /mx
+      end
+      memoize :build_regex
+
+      # Parse YAML frontmatter out of a string
+      # @param [String] content
+      # @return [Hash]
+      Contract String, Pathname => Hash
+      def parse_yaml(content, full_path)
+        c = begin
+          ::Middleman::Util.instrument 'parse.yaml' do
+            ::YAML.load(content)
+          end
+        rescue StandardError, ::Psych::SyntaxError => error
+          warn "YAML Exception parsing #{full_path}: #{error.message}"
+          {}
+        end
+      
+        c ? symbolize_recursive(c) : {}
+      end
+      memoize :parse_yaml
+      
+      # Parse TOML frontmatter out of a string
+      # @param [String] content
+      # @return [Hash]
+      Contract String, Pathname => Hash
+      def parse_toml(content, full_path)
+        c = begin
+          ::Middleman::Util.instrument 'parse.toml' do
+            ::TOML.load(content)
+          end
+        rescue StandardError
+          # TOML parser swallows useful error, so we can't warn about it.
+          # https://github.com/jm/toml/issues/47
+          warn "TOML Exception parsing #{full_path}"
+          {}
+        end
+      
+        c ? symbolize_recursive(c) : {}
+      end
+      memoize :parse_yaml
+
+      # Parse JSON frontmatter out of a string
+      # @param [String] content
+      # @return [Hash]
+      Contract String, Pathname => Hash
+      def parse_json(content, full_path)
+        c = begin
+          ::Middleman::Util.instrument 'parse.json' do
+            ::JSON.parse(content)
+          end
+        rescue StandardError => error
+          warn "JSON Exception parsing #{full_path}: #{error.message}"
+          {}
+        end
+
+        c ? symbolize_recursive(c) : {}
+      end
+      memoize :parse_json
+
+      def symbolize_recursive(value)
+        case value
+        when Hash
+          value.map do |k, v|
+            key = k.is_a?(String) ? k.to_sym : k
+            [key, symbolize_recursive(v)]
+          end.to_h
+        when Array
+          value.map { |v| symbolize_recursive(v) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/middleman-core/util/files.rb

@@ -0,0 +1,137 @@
+require 'set'
+
+module Middleman
+  module Util
+    include Contracts
+
+    module_function
+
+    # Get a recusive list of files inside a path.
+    # Works with symlinks.
+    #
+    # @param path Some path string or Pathname
+    # @param ignore A proc/block that returns true if a given path should be ignored - if a path
+    #               is ignored, nothing below it will be searched either.
+    # @return [Array<Pathname>] An array of Pathnames for each file (no directories)
+    Contract Or[String, Pathname], Proc => ArrayOf[Pathname]
+    def all_files_under(path, &ignore)
+      path = Pathname(path)
+
+      if ignore && yield(path)
+        []
+      elsif path.directory?
+        path.children.flat_map do |child|
+          all_files_under(child, &ignore)
+        end.compact
+      elsif path.file?
+        [path]
+      else
+        []
+      end
+    end
+
+    # Glob a directory and try to keep path encoding consistent.
+    #
+    # @param [String] path The glob path.
+    # @return [Array<String>]
+    def glob_directory(path)
+      results = ::Dir[path]
+
+      return results unless RUBY_PLATFORM =~ /darwin/
+
+      results.map { |r| r.encode('UTF-8', 'UTF-8-MAC') }
+    end
+
+    # Get the PWD and try to keep path encoding consistent.
+    #
+    # @param [String] path The glob path.
+    # @return [Array<String>]
+    def current_directory
+      result = ::Dir.pwd
+
+      return result unless RUBY_PLATFORM =~ /darwin/
+
+      result.encode('UTF-8', 'UTF-8-MAC')
+    end
+
+    Contract String => String
+    def step_through_extensions(path)
+      while ::Middleman::Util.tilt_class(path)
+        ext = ::File.extname(path)
+        break if ext.empty?
+
+        yield ext if block_given?
+
+        # Strip templating extensions as long as Tilt knows them
+        path = path[0..-(ext.length + 1)]
+      end
+
+      yield ::File.extname(path) if block_given?
+
+      path
+    end
+
+    # Removes the templating extensions, while keeping the others
+    # @param [String] path
+    # @return [String]
+    Contract String => String
+    def remove_templating_extensions(path)
+      step_through_extensions(path)
+    end
+
+    # Removes the templating extensions, while keeping the others
+    # @param [String] path
+    # @return [String]
+    Contract String => ArrayOf[String]
+    def collect_extensions(path)
+      @@extensions_cache ||= {}
+
+      base_name = ::File.basename(path)
+      @@extensions_cache[base_name] ||= begin
+        result = []
+
+        unless base_name.start_with?('.')
+          step_through_extensions(base_name) { |e| result << e }
+        end
+
+        result
+      end
+    end
+
+    # Finds files which should also be considered to be dirty when
+    # the given file(s) are touched.
+    #
+    # @param [Middleman::Application] app The app.
+    # @param [Pathname] files The original touched file paths.
+    # @return [Middleman::SourceFile] All related file paths, not including the source file paths.
+    Contract ::Middleman::Application, ArrayOf[Pathname] => ArrayOf[::Middleman::SourceFile]
+    def find_related_files(app, files)
+      return [] if files.empty?
+
+      file_set = ::Set.new(files)
+
+      all_extensions = files.flat_map { |f| collect_extensions(f.to_s) }
+      sass_type_aliasing = ['.scss', '.sass']
+      erb_type_aliasing = ['.erb', '.haml', '.slim']
+
+      all_extensions |= sass_type_aliasing unless (all_extensions & sass_type_aliasing).empty?
+      all_extensions |= erb_type_aliasing unless (all_extensions & erb_type_aliasing).empty?
+
+      all_extensions.uniq!
+
+      app.sitemap.resources.select { |r|
+        if r.file_descriptor
+          local_extensions = collect_extensions(r.file_descriptor[:full_path].to_s)
+          local_extensions |= sass_type_aliasing unless (local_extensions & sass_type_aliasing).empty?
+          local_extensions |= erb_type_aliasing unless (local_extensions & erb_type_aliasing).empty?
+
+          local_extensions.uniq!
+
+          !(all_extensions & local_extensions).empty? && !file_set.include?(r.file_descriptor[:full_path])
+        else
+          false
+        end
+      }.map(&:file_descriptor)
+    end
+  end
+end

data/lib/middleman-core/util/paths.rb

@@ -0,0 +1,313 @@
+# Core Pathname library used for traversal
+require 'pathname'
+require 'uri'
+require 'addressable/uri'
+require 'memoist'
+require 'tilt'
+require 'webrick'
+
+require 'middleman-core/contracts'
+
+# rubocop:disable ModuleLength
+module Middleman
+  module Util
+    extend Memoist
+    include Contracts
+
+    module_function
+
+    Contract String => ::Addressable::URI
+    def parse_uri(uri)
+      ::Addressable::URI.parse(uri)
+    end
+    memoize :parse_uri
+
+    Contract String => Any
+    def tilt_class(path)
+      ::Tilt[path]
+    end
+    memoize :tilt_class
+
+    # Normalize a path to not include a leading slash
+    # @param [String] path
+    # @return [String]
+    Contract String => String
+    def normalize_path(path)
+      # The tr call works around a bug in Ruby's Unicode handling
+      WEBrick::HTTPUtils.unescape(path).sub(%r{^/}, '').tr('', '')
+    end
+    memoize :normalize_path
+
+    # This is a separate method from normalize_path in case we
+    # change how we normalize paths
+    Contract String => String
+    def strip_leading_slash(path)
+      path.sub(%r{^/}, '')
+    end
+    memoize :strip_leading_slash
+
+    IGNORE_DESCRIPTOR = Or[Regexp, RespondTo[:call], String]
+    Contract IGNORE_DESCRIPTOR, String => Bool
+    def should_ignore?(validator, value)
+      if validator.is_a? Regexp
+        # Treat as Regexp
+        !!(value =~ validator)
+      elsif validator.respond_to? :call
+        # Treat as proc
+        validator.call(value)
+      elsif validator.is_a? String
+        # Treat as glob
+        File.fnmatch(value, validator)
+      else
+        # If some unknown thing, don't ignore
+        false
+      end
+    end
+    memoize :should_ignore?
+
+    # Get the path of a file of a given type
+    #
+    # @param [Middleman::Application] app The app.
+    # @param [Symbol] kind The type of file
+    # @param [String, Symbol] source The path to the file
+    # @param [Hash] options Data to pass through.
+    # @return [String]
+    Contract ::Middleman::Application, Symbol, Or[String, Symbol], Hash => String
+    def asset_path(app, kind, source, options={})
+      return source if source.to_s.include?('//') || source.to_s.start_with?('data:')
+
+      asset_folder = case kind
+      when :css
+        app.config[:css_dir]
+      when :js
+        app.config[:js_dir]
+      when :images
+        app.config[:images_dir]
+      when :fonts
+        app.config[:fonts_dir]
+      else
+        kind.to_s
+      end
+
+      source = source.to_s.tr(' ', '')
+      ignore_extension = (kind == :images || kind == :fonts) # don't append extension
+      source << ".#{kind}" unless ignore_extension || source.end_with?(".#{kind}")
+      asset_folder = '' if source.start_with?('/') # absolute path
+
+      asset_url(app, source, asset_folder, options)
+    end
+
+    # Get the URL of an asset given a type/prefix
+    #
+    # @param [String] path The path (such as "photo.jpg")
+    # @param [String] prefix The type prefix (such as "images")
+    # @param [Hash] options Data to pass through.
+    # @return [String] The fully qualified asset url
+    Contract ::Middleman::Application, String, String, Hash => String
+    def asset_url(app, path, prefix='', options={})
+      # Don't touch assets which already have a full path
+      return path if path.include?('//') || path.start_with?('data:')
+
+      if options[:relative] && !options[:current_resource]
+        raise ArgumentError, '#asset_url must be run in a context with current_resource if relative: true'
+      end
+
+      uri = ::Middleman::Util.parse_uri(path)
+      path = uri.path
+
+      # Ensure the url we pass into find_resource_by_destination_path is not a
+      # relative path, since it only takes absolute url paths.
+      dest_path = url_for(app, path, options.merge(relative: false))
+
+      result = if resource = app.sitemap.find_resource_by_path(dest_path)
+        resource.url
+      elsif resource = app.sitemap.find_resource_by_destination_path(dest_path)
+        resource.url
+      else
+        path = ::File.join(prefix, path)
+        if resource = app.sitemap.find_resource_by_path(path)
+          resource.url
+        else
+          ::File.join(app.config[:http_prefix], path)
+        end
+      end
+
+      final_result = ::Addressable::URI.encode(
+        relative_path_from_resource(
+          options[:current_resource],
+          result,
+          options[:relative]
+        )
+      )
+
+      result_uri = ::Middleman::Util.parse_uri(final_result)
+      result_uri.query = uri.query
+      result_uri.fragment = uri.fragment
+      result_uri.to_s
+    end
+
+    # Given a source path (referenced either absolutely or relatively)
+    # or a Resource, this will produce the nice URL configured for that
+    # path, respecting :relative_links, directory indexes, etc.
+    Contract ::Middleman::Application, Or[String, Symbol, ::Middleman::Sitemap::Resource], Hash => String
+    def url_for(app, path_or_resource, options={})
+      if path_or_resource.is_a?(String) || path_or_resource.is_a?(Symbol)
+        r = app.sitemap.find_resource_by_page_id(path_or_resource)
+
+        path_or_resource = r ? r : path_or_resource.to_s
+      end
+
+      # Handle Resources and other things which define their own url method
+      url = if path_or_resource.respond_to?(:url)
+        path_or_resource.url
+      else
+        path_or_resource.dup
+      end
+
+      # Try to parse URL
+      begin
+        uri = ::Middleman::Util.parse_uri(url)
+      rescue ::Addressable::URI::InvalidURIError
+        # Nothing we can do with it, it's not really a URI
+        return url
+      end
+
+      relative = options[:relative]
+      raise "Can't use the relative option with an external URL" if relative && uri.host
+
+      # Allow people to turn on relative paths for all links with
+      # set :relative_links, true
+      # but still override on a case by case basis with the :relative parameter.
+      effective_relative = relative || false
+      effective_relative = true if relative.nil? && app.config[:relative_links]
+
+      # Try to find a sitemap resource corresponding to the desired path
+      this_resource = options[:current_resource]
+
+      if path_or_resource.is_a?(::Middleman::Sitemap::Resource)
+        resource = path_or_resource
+        resource_url = url
+      elsif this_resource && uri.path && !uri.host
+        # Handle relative urls
+        url_path = Pathname(uri.path)
+        current_source_dir = Pathname('/' + this_resource.path).dirname
+        url_path = current_source_dir.join(url_path) if url_path.relative?
+        resource = app.sitemap.find_resource_by_path(url_path.to_s)
+        if resource
+          resource_url = resource.url
+        else
+          # Try to find a resource relative to destination paths
+          url_path = Pathname(uri.path)
+          current_source_dir = Pathname('/' + this_resource.destination_path).dirname
+          url_path = current_source_dir.join(url_path) if url_path.relative?
+          resource = app.sitemap.find_resource_by_destination_path(url_path.to_s)
+          resource_url = resource.url if resource
+        end
+      elsif options[:find_resource] && uri.path && !uri.host
+        resource = app.sitemap.find_resource_by_path(uri.path)
+        resource_url = resource.url if resource
+      end
+
+      if resource
+        uri.path = if this_resource
+          ::Addressable::URI.encode(
+            relative_path_from_resource(
+              this_resource,
+              resource_url,
+              effective_relative
+            )
+          )
+        else
+          resource_url
+        end
+      end
+
+      # Support a :query option that can be a string or hash
+      if query = options[:query]
+        uri.query = query.respond_to?(:to_param) ? query.to_param : query.to_s
+      end
+
+      # Support a :fragment or :anchor option just like Padrino
+      fragment = options[:anchor] || options[:fragment]
+      uri.fragment = fragment.to_s if fragment
+
+      # Finally make the URL back into a string
+      uri.to_s
+    end
+
+    # Expand a path to include the index file if it's a directory
+    #
+    # @param [String] path Request path/
+    # @param [Middleman::Application] app The requesting app.
+    # @return [String] Path with index file if necessary.
+    Contract String, ::Middleman::Application => String
+    def full_path(path, app)
+      resource = app.sitemap.find_resource_by_destination_path(path)
+
+      unless resource
+        # Try it with /index.html at the end
+        indexed_path = ::File.join(path.sub(%r{/$}, ''), app.config[:index_file])
+        resource = app.sitemap.find_resource_by_destination_path(indexed_path)
+      end
+
+      if resource
+        '/' + resource.destination_path
+      else
+        '/' + normalize_path(path)
+      end
+    end
+
+    # Get a relative path to a resource.
+    #
+    # @param [Middleman::Sitemap::Resource] curr_resource The resource.
+    # @param [String] resource_url The target url.
+    # @param [Boolean] relative If the path should be relative.
+    # @return [String]
+    Contract ::Middleman::Sitemap::Resource, String, Bool => String
+    def relative_path_from_resource(curr_resource, resource_url, relative)
+      # Switch to the relative path between resource and the given resource
+      # if we've been asked to.
+      if relative
+        # Output urls relative to the destination path, not the source path
+        current_dir = Pathname('/' + curr_resource.destination_path).dirname
+        relative_path = Pathname(resource_url).relative_path_from(current_dir).to_s
+
+        # Put back the trailing slash to avoid unnecessary Apache redirects
+        if resource_url.end_with?('/') && !relative_path.end_with?('/')
+          relative_path << '/'
+        end
+
+        relative_path
+      else
+        resource_url
+      end
+    end
+
+    # Takes a matcher, which can be a literal string
+    # or a string containing glob expressions, or a
+    # regexp, or a proc, or anything else that responds
+    # to #match or #call, and returns whether or not the
+    # given path matches that matcher.
+    #
+    # @param [String, #match, #call] matcher A matcher String, RegExp, Proc, etc.
+    # @param [String] path A path as a string
+    # @return [Boolean] Whether the path matches the matcher
+    Contract PATH_MATCHER, String => Bool
+    def path_match(matcher, path)
+      if matcher.is_a?(String)
+        if matcher.include? '*'
+          ::File.fnmatch(matcher, path)
+        else
+          path == matcher
+        end
+      elsif matcher.respond_to?(:match)
+        !!(path =~ matcher)
+      elsif matcher.respond_to?(:call)
+        matcher.call(path)
+      else
+        ::File.fnmatch(matcher.to_s, path)
+      end
+    end
+    memoize :path_match
+  end
+end