middleman-core 4.5.0 → 5.0.0.rc.1

data/lib/middleman-core/util/binary.rb

@@ -1,9 +1,271 @@
 # Template and Mime detection
 require 'tilt'
 require 'rack/mime'
+require 'set'
 
 require 'middleman-core/contracts'
 
+KNOWN_NON_STATIC_FILE_EXTENSIONS = Set.new %w[
+  css
+  js
+  mjs
+  html
+  htm
+  php
+]
+
+KNOWN_BINARY_FILE_EXTENSIONS = Set.new %w[
+  3dm
+  3ds
+  3g2
+  3gp
+  7z
+  a
+  aac
+  adp
+  ai
+  aif
+  aiff
+  alz
+  ape
+  apk
+  ar
+  arj
+  asf
+  au
+  avi
+  bak
+  baml
+  bh
+  bin
+  bk
+  bmp
+  btif
+  bz2
+  bzip2
+  cab
+  caf
+  cgm
+  class
+  cmx
+  cpio
+  cr2
+  csv
+  cur
+  dat
+  dcm
+  deb
+  dex
+  djvu
+  dll
+  dmg
+  dng
+  doc
+  docm
+  docx
+  dot
+  dotm
+  dra
+  DS_Store
+  dsk
+  dts
+  dtshd
+  dvb
+  dwg
+  dxf
+  ecelp4800
+  ecelp7470
+  ecelp9600
+  egg
+  eol
+  eot
+  epub
+  exe
+  f4v
+  fbs
+  fh
+  fla
+  flac
+  fli
+  flv
+  fpx
+  fst
+  fvt
+  g3
+  gh
+  gif
+  graffle
+  gz
+  gzip
+  h261
+  h263
+  h264
+  icns
+  ico
+  ief
+  img
+  ipa
+  iso
+  jar
+  jpeg
+  jpg
+  jpgv
+  jpm
+  jxr
+  key
+  ktx
+  lha
+  lib
+  lvp
+  lz
+  lzh
+  lzma
+  lzo
+  m3u
+  m4a
+  m4v
+  mar
+  mdi
+  mht
+  mid
+  midi
+  mj2
+  mka
+  mkv
+  mmr
+  mng
+  mobi
+  mov
+  movie
+  mp3
+  mp4
+  mp4a
+  mpeg
+  mpg
+  mpga
+  mxu
+  nef
+  npx
+  numbers
+  o
+  oga
+  ogg
+  ogv
+  otf
+  pages
+  pbm
+  pcx
+  pdb
+  pdf
+  pea
+  pgm
+  pic
+  png
+  pnm
+  pot
+  potm
+  potx
+  ppa
+  ppam
+  ppm
+  pps
+  ppsm
+  ppsx
+  ppt
+  pptm
+  pptx
+  psd
+  pya
+  pyc
+  pyo
+  pyv
+  qt
+  rar
+  ras
+  raw
+  resources
+  rgb
+  rip
+  rlc
+  rmf
+  rmvb
+  rtf
+  rz
+  s3m
+  s7z
+  scpt
+  sgi
+  shar
+  sil
+  sketch
+  slk
+  smv
+  so
+  stl
+  sub
+  svgz
+  swf
+  tar
+  tbz
+  tbz2
+  tga
+  tgz
+  thmx
+  tif
+  tiff
+  tlz
+  ttc
+  ttf
+  txz
+  udf
+  uvh
+  uvi
+  uvm
+  uvp
+  uvs
+  uvu
+  viv
+  vob
+  war
+  wav
+  wax
+  wbmp
+  wdp
+  weba
+  webm
+  webp
+  whl
+  wim
+  wm
+  wma
+  wmv
+  wmx
+  woff
+  woff2
+  wrm
+  wvx
+  xbm
+  xif
+  xla
+  xlam
+  xls
+  xlsb
+  xlsm
+  xlsx
+  xlt
+  xltm
+  xltx
+  xm
+  xmind
+  xpi
+  xpm
+  xwd
+  xz
+  z
+  zip
+  zipx
+]
+
 module Middleman
   module Util
     include Contracts
@@ -23,16 +285,19 @@ module Middleman
       @binary_cache[filename] = begin
         path = Pathname(filename)
         ext = path.extname
+        without_dot = ext.sub('.', '')
 
         # We hardcode detecting of gzipped SVG files
-        if ext == '.svgz'
+        if KNOWN_BINARY_FILE_EXTENSIONS.include?(without_dot)
           true
-        elsif ::Tilt.registered?(ext.sub('.', ''))
+        elsif ::Tilt.registered?(without_dot)
           false
         else
           dot_ext = ext.to_s[0] == '.' ? ext.dup : ".#{ext}"
 
-          if mime = ::Rack::Mime.mime_type(dot_ext, nil)
+          mime = ::Rack::Mime.mime_type(dot_ext, nil)
+
+          if mime
             !nonbinary_mime?(mime)
           else
             file_contents_include_binary_bytes?(path.to_s)
@@ -74,5 +339,44 @@ module Middleman
 
       false
     end
+
+    Contract String, Maybe[HashOf[Symbol, Any]] => Bool
+    def static_file?(path, frontmatter_delims)
+      path = Pathname(path)
+      ext = path.extname
+      without_dot = ext.sub('.', '')
+
+      if KNOWN_NON_STATIC_FILE_EXTENSIONS.include?(without_dot) || contains_frontmatter?(path, frontmatter_delims)
+        false
+      else
+        !::Tilt.registered?(without_dot)
+      end
+    end
+
+    Contract String, Maybe[HashOf[Symbol, Any]] => Bool
+    def contains_frontmatter?(path, frontmatter_delims)
+      file = ::File.open(path)
+      first_line = file.gets
+
+      first_line = file.gets if first_line =~ /\A(?:[^\r\n]*coding:[^\r\n]*\r?\n)/
+
+      file.close
+
+      possible_openers = possible_delim_openers(frontmatter_delims)
+      !first_line.nil? && !first_line.match(possible_openers).nil?
+    rescue EOFError, IOError, ::Errno::ENOENT
+      false
+    end
+
+    Contract Maybe[HashOf[Symbol, Any]] => Regexp
+    def possible_delim_openers(frontmatter_delims)
+      all_possible = frontmatter_delims
+                     .values
+                     .flatten(1)
+                     .map(&:first)
+                     .uniq
+
+      /\A#{::Regexp.union(all_possible)}/
+    end
   end
 end

data/lib/middleman-core/util/data.rb

@@ -1,8 +1,6 @@
 require 'yaml'
 require 'json'
-require 'toml'
 require 'pathname'
-require 'backports/2.1.0/array/to_h'
 require 'hashie'
 require 'memoist'
 
@@ -26,7 +24,7 @@ module Middleman
     # @private
     # @param [Hash] data Normal hash
     # @return [Hash]
-    Contract Any => Maybe[Or[Array, EnhancedHash]]
+    Contract Hash => IsA['::Middleman::Util::EnhancedHash']
     def recursively_enhance(obj)
       if obj.is_a? ::Array
         obj.map { |e| recursively_enhance(e) }
@@ -47,13 +45,14 @@ module Middleman
       # @param [String] path
       # @return [Array<Hash, String>]
       Contract IsA['Middleman::SourceFile'], Maybe[Symbol] => [Hash, Maybe[String]]
-      def parse(file, frontmatter_delims, known_type=nil)
-        full_path = file[:full_path]
-        return [{}, nil] if ::Middleman::Util.binary?(full_path) || file[:types].include?(:binary)
+      def parse(source_file, frontmatter_delims, known_type = nil)
+        full_path = source_file[:full_path]
+
+        return [{}, nil] if ::Middleman::Util.binary?(full_path) || source_file[:types].include?(:binary)
 
         # Avoid weird race condition when a file is renamed
         begin
-          content = file.read
+          content = source_file.read
         rescue EOFError, IOError, ::Errno::ENOENT
           return [{}, nil]
         end
@@ -66,8 +65,6 @@ module Middleman
             return [parse_yaml(content, full_path), nil]
           when :json
             return [parse_json(content, full_path), nil]
-          when :toml
-            return [parse_toml(content, full_path), nil]
           end
         end
 
@@ -82,11 +79,6 @@ module Middleman
             parse_json("{#{match[:frontmatter]}}", full_path),
             match[:additional_content]
           ]
-        when *frontmatter_delims[:toml]
-          [
-            parse_toml(match[:frontmatter], full_path),
-            match[:additional_content]
-          ]
         else
           [
             {},
@@ -118,43 +110,15 @@ module Middleman
       # @return [Hash]
       Contract String, Pathname => Hash
       def parse_yaml(content, full_path)
-        permitted_classes = [Date, Symbol]
         c = begin
           ::Middleman::Util.instrument 'parse.yaml' do
-            allowed_parameters = ::YAML.method(:safe_load).parameters
-            if allowed_parameters.include? [:key, :permitted_classes]
-              ::YAML.safe_load(content, permitted_classes: permitted_classes)
-            elsif allowed_parameters.include? [:key, :whitelist_classes]
-              ::YAML.safe_load(content, whitelist_classes: permitted_classes)
-            else
-              ::YAML.safe_load(content, permitted_classes)
-            end
-          end
-        rescue StandardError, ::Psych::SyntaxError => error
-          warn "YAML Exception parsing #{full_path}: #{error.message}"
-          {}
-        end
-      
-        c ? symbolize_recursive(c) : {}
-      end
-      memoize :parse_yaml
-      
-      # Parse TOML frontmatter out of a string
-      # @param [String] content
-      # @return [Hash]
-      Contract String, Pathname => Hash
-      def parse_toml(content, full_path)
-        c = begin
-          ::Middleman::Util.instrument 'parse.toml' do
-            ::TOML.load(content)
+            ::YAML.load(content)
           end
-        rescue StandardError
-          # TOML parser swallows useful error, so we can't warn about it.
-          # https://github.com/jm/toml/issues/47
-          warn "TOML Exception parsing #{full_path}"
-          {}
+            rescue StandardError, ::Psych::SyntaxError => error
+              warn "YAML Exception parsing #{full_path}: #{error.message}"
+              {}
         end
-      
+
         c ? symbolize_recursive(c) : {}
       end
       memoize :parse_yaml
@@ -168,9 +132,9 @@ module Middleman
           ::Middleman::Util.instrument 'parse.json' do
             ::JSON.parse(content)
           end
-        rescue StandardError => error
-          warn "JSON Exception parsing #{full_path}: #{error.message}"
-          {}
+            rescue StandardError => error
+              warn "JSON Exception parsing #{full_path}: #{error.message}"
+              {}
         end
 
         c ? symbolize_recursive(c) : {}

data/lib/middleman-core/util/empty_hash.rb

@@ -0,0 +1,3 @@
+module Middleman
+  EMPTY_HASH = {}.freeze
+end

data/lib/middleman-core/util/files.rb

@@ -1,4 +1,7 @@
+require 'digest/sha1'
 require 'set'
+require 'open3'
+require 'shellwords'
 
 module Middleman
   module Util
@@ -6,7 +9,7 @@ module Middleman
 
     module_function
 
-    # Get a recusive list of files inside a path.
+    # Get a recursive list of files inside a path.
     # Works with symlinks.
     #
     # @param path Some path string or Pathname
@@ -56,9 +59,8 @@ module Middleman
 
     Contract String => String
     def step_through_extensions(path)
-      while ::Middleman::Util.tilt_class(path)
-        ext = ::File.extname(path)
-        break if ext.empty?
+      while (ext = File.extname(path))
+        break if ext.empty? || !::Middleman::Util.tilt_class(ext)
 
         yield ext if block_given?
 
@@ -90,9 +92,7 @@ module Middleman
       @@extensions_cache[base_name] ||= begin
         result = []
 
-        unless base_name.start_with?('.')
-          step_through_extensions(base_name) { |e| result << e }
-        end
+        step_through_extensions(base_name) { |e| result << e } unless base_name.start_with?('.')
 
         result
       end
@@ -119,7 +119,7 @@ module Middleman
 
       all_extensions.uniq!
 
-      app.sitemap.resources.select { |r|
+      app.sitemap.without_ignored.to_a.select do |r|
         if r.file_descriptor
           local_extensions = collect_extensions(r.file_descriptor[:full_path].to_s)
           local_extensions |= sass_type_aliasing unless (local_extensions & sass_type_aliasing).empty?
@@ -131,7 +131,33 @@ module Middleman
         else
           false
         end
-      }.map(&:file_descriptor)
+      end.map(&:file_descriptor)
+    end
+
+    Contract String, Maybe[Num] => String
+    def read_file(path, bytes = nil)
+      # puts "Read: #{path}"
+      File.read(path, bytes)
+    end
+
+    Contract String => String
+    def hash_file(path)
+      # puts "Read (hash): #{path}"
+
+      if ENV['MIDDLEMAN_SHELL_OUT_TO_GIT_HASH'] == 'true'
+        output, status = ::Open3.capture2e("git hash-object #{::Shellwords.escape(path)}")
+
+        raise "Failed to get hash for '#{path}' from git." if status.exitstatus != 0 || output.empty?
+
+        output.strip
+      else
+        ::Digest::SHA1.file(path).hexdigest
+      end
+    end
+
+    Contract String => String
+    def hash_string(data)
+      ::Digest::SHA1.hexdigest(data)
     end
   end
 end