slinky 0.7.3 → 0.8.0

data/lib/slinky/graph.rb

@@ -0,0 +1,113 @@
+# coding: utf-8
+module Slinky
+  # The Graph class describes a directed graph and provides various
+  # graph algorithms.
+  class Graph
+    include Enumerable
+
+    attr_reader :nodes, :edges
+
+    # Creates a new Graph from an adjacency list
+    def initialize nodes, edges
+      @nodes = nodes
+      @edges = edges
+    end
+
+    # Builds an adjacency matrix representation of the graph
+    def adjacency_matrix
+      return @adjacency_matrix if @adjacency_matrix
+
+      # Convert from adjacency list to a map structure
+      g = Hash.new{|h,k| h[k] = []}
+      edges.each{|x|
+        g[x[1]] << x[0]
+      }
+
+      @adjacency_matrix = g
+    end
+
+    # Builds the transitive closure of the dependency graph using
+    # Floyd–Warshall
+    def transitive_closure
+      return @transitive_closure if @transitive_closure
+
+      g = adjacency_matrix
+      
+      index_map = {}
+      nodes.each_with_index{|f, i| index_map[f] = i}
+
+      size = nodes.size
+
+      # Set up the distance matrix
+      dist = Array.new(size){|_| Array.new(size, Float::INFINITY)}
+      nodes.each_with_index{|fi, i|
+        dist[i][i] = 0
+        g[fi].each{|fj|
+          dist[i][index_map[fj]] = 1
+        }
+      }
+
+      # Compute the all-paths costs
+      size.times{|k|
+        size.times{|i|
+          size.times{|j|
+            if dist[i][j] > dist[i][k] + dist[k][j] 
+              dist[i][j] = dist[i][k] + dist[k][j]
+            end
+          }
+        }
+      }
+
+      # Compute the transitive closure in map form
+      @transitive_closure = Hash.new{|h,k| h[k] = []}
+      size.times{|i|
+        size.times{|j|
+          if dist[i][j] < Float::INFINITY
+            @transitive_closure[nodes[i]] << nodes[j]
+          end
+        }
+      }
+
+      @transitive_closure
+    end
+
+    # Builds a list of files in topological order, so that when
+    # required in this order all dependencies are met. See
+    # http://en.wikipedia.org/wiki/Topological_sorting for more
+    # information.
+    def dependency_list
+      return @dependency_list if @dependency_list
+
+      graph = edges.clone
+      # will contain sorted elements
+      l = []
+      # start nodes, those with no incoming edges
+      s = nodes.reject{|mf| mf.directives[:slinky_require]}
+      while s.size > 0
+        n = s.delete s.first
+        l << n
+        nodes.each{|m|
+          e = graph.find{|e| e[0] == n && e[1] == m}
+          next unless e
+          graph.delete e
+          s << m unless graph.any?{|e| e[1] == m}
+        }
+      end
+      if graph != []
+        problems = graph.collect{|e| e.collect{|x| x.source}.join(" -> ")}
+        raise DependencyError.new("Dependencies #{problems.join(", ")} could not be satisfied")
+      end
+      @dependency_list = l
+    end
+
+    def each &block  
+      edges.each do |e|
+        if block_given?
+          block.call e
+        else  
+          yield e
+        end
+      end  
+    end
+  end
+end

data/lib/slinky/manifest.rb

@@ -1,30 +1,26 @@
 # -*- coding: utf-8 -*-
 require 'pathname'
 require 'digest/md5'
+require 'matrix'
+require 'set'
 
 module Slinky
-  # extensions of files that can contain build directives
-  DIRECTIVE_FILES = %w{js css html haml sass scss coffee}
+  # extensions of non-compiled files that can contain build directives
+  DIRECTIVE_FILES = %w{js css html}
   DEPENDS_DIRECTIVE = /^[^\n\w]*(slinky_depends)\((".*"|'.+'|)\)[^\n\w]*$/
   REQUIRE_DIRECTIVE = /^[^\n\w]*(slinky_require)\((".*"|'.+'|)\)[^\n\w]*$/
   SCRIPTS_DIRECTIVE = /^[^\n\w]*(slinky_scripts)[^\n\w]*$/
   STYLES_DIRECTIVE  = /^[^\n\w]*(slinky_styles)[^\n\w]*$/
+  PRODUCT_DIRECTIVE = /^[^\n\w]*(slinky_product)\((".*"|'.+'|)\)[^\n\w]*$/
   BUILD_DIRECTIVES = Regexp.union(DEPENDS_DIRECTIVE,
                                   REQUIRE_DIRECTIVE,
                                   SCRIPTS_DIRECTIVE,
-                                  STYLES_DIRECTIVE)
+                                  STYLES_DIRECTIVE,
+                                  PRODUCT_DIRECTIVE)
   CSS_URL_MATCHER = /url\(['"]?([^'"\/][^\s)]+\.[a-z]+)(\?\d+)?['"]?\)/
 
-  # Raised when a compilation fails for any reason
-  class BuildFailedError < StandardError; end
-  # Raised when a required file is not found.
-  class FileNotFoundError < StandardError; end
-  # Raised when there is a cycle in the dependency graph (i.e., file A
-  # requires file B which requires C which requires A)
-  class DependencyError < StandardError; end
-
   class Manifest
-    attr_accessor :manifest_dir, :dir
+    attr_accessor :manifest_dir, :dir, :config
 
     def initialize dir, config, options = {}
       @dir = dir
@@ -87,59 +83,170 @@ module Slinky
       @manifest_dir.find_by_path path, allow_multiple
     end
 
-    def scripts_string
-      if @devel
-        dependency_list.reject{|x| x.output_path.extname != ".js" }.collect{|d|
-          %Q\<script type="text/javascript" src="/#{d.relative_output_path}"></script>\
-        }.join("\n")
-      else
-        %Q\<script type="text/javascript" src="/scripts.js?#{rand(999999999)}"></script>\
+    # Finds all files that match the given pattern. The match rules
+    # are similar to those for .gitignore and given by
+    #
+    # 1. If the pattern ends with a slash, it will only match directories;
+    #    e.g. `foo/` would match a directory `foo/` but not a file `foo`. In
+    #    a file context, matching a directory is equivalent to matching all
+    #    files under that directory, recursively.
+    # 2. If the pattern does not contain a slash, slinky treats it as a
+    #    relative pathname which can match files in any directory. For
+    #    example, the rule `test.js` will matching `/test.js` and
+    #   `/component/test.js`.
+    # 3. If the pattern begins with a slash, it will be treated as an
+    #    absolute path starting at the root of the source directory.
+    # 4. If the pattern does not begin with a slash, but does contain one or
+    #    more slashes, it will be treated as a path relative to any
+    #    directory. For example, `test/*.js` will match `/test/main.js`, and
+    #    /component/test/component.js`, but not `main.js`.
+    # 5. A single star `*` in a pattern will match any number of characters within a
+    #    single path component. For example, `/test/*.js` will match
+    #    `/test/main_test.js` but not `/test/component/test.js`.
+    # 6. A double star `**` will match any number of characters including
+    #    path separators. For example `/scripts/**/main.js` will match any
+    #    file named `main.js` under the `/scripts` directory, including
+    #   `/scripts/main.js` and `/scripts/component/main.js`.
+    def find_by_pattern pattern
+      # The strategy here is to convert the pattern into an equivalent
+      # regex and run that against the pathnames of all the files in
+      # the manifest.
+      regex_str = Regexp.escape(pattern)
+                  .gsub('\*\*/', ".*")
+                  .gsub('\*\*', ".*")
+                  .gsub('\*', "[^/]*")
+
+      if regex_str[0] != '/'
+        regex_str = '.*/' + regex_str
+      end
+
+      if regex_str[-1] == '/'
+        regex_str += '.*'
       end
+
+      regex_str = "^#{regex_str}$"
+
+      regex = Regexp.new(regex_str)
+
+      files(false).reject{|f|
+        !regex.match('/' + f.relative_source_path.to_s) &&
+        !regex.match('/' + f.relative_output_path.to_s)
+      }
     end
 
-    def compress ext, output, compressor
-      scripts = dependency_list.reject{|x| x.output_path.extname != ext}
+    # Finds all the matching manifest files for a particular product.
+    # This does not take into account dependencies.
+    def files_for_product product
+      if !p = @config.produce[product]
+        SlinkyError.raise NoSuchProductError,
+               "Product '#{product}' has not been configured"
+      end
 
-      if scripts.size > 0
-        s = scripts.collect{|s|
-          f = File.open(s.build_to.to_s, 'rb'){|f| f.read}
-          (block_given?) ? (yield s, f) : f
-        }.join("\n")
+      type = type_for_product product
+      if type != ".js" && type != ".css"
+        SlinkyError.raise InvalidConfigError, "Only .js and .css products are supported"
+      end
 
-        File.open(output, "w+"){|f|
-          unless @no_minify
-            f.write(compressor.compress(s))
-          else
-            f.write(s)
+      g = dependency_graph.transitive_closure
+
+      # Topological indices for each file
+      indices = {}
+      dependency_list.each_with_index{|f, i| indices[f] = i}
+
+      # Compute the set of excluded files
+      excludes = Set.new((p["exclude"] || []).map{|p|
+                           find_by_pattern(p)
+                         }.flatten.uniq)
+
+      SlinkyError.batch_errors do
+        # First find the list of files that have been explictly
+        # included/excluded
+        p["include"].map{|f|
+          mfs = find_by_pattern(f)
+                .map{|mf| [mf] + g[f]}
+                .flatten
+                .reject{|f| f.output_path.extname != type}
+          if mfs.empty?
+            SlinkyError.raise FileNotFoundError,
+                              "No files matched by include #{f} in product #{product}"
           end
+          mfs.flatten
+        }.flatten.reject{|f|
+          excludes.include?(f)
+          # Then add all the files these require
+        }.map{|f|
+          # Find all of the downstream files
+          # check that we're not excluding any required files
+          g[f].each{|rf|
+            if p["exclude"] && r = p["exclude"].find{|ex| rf.matches_path?(ex, true)}
+              SlinkyError.raise DependencyError,
+                "File #{f} requires #{rf} which is excluded by exclusion rule #{r}"
+            end
+          }
+          [f] + g[f]
+        }.flatten.uniq.sort_by{|f|
+          # Sort by topological order
+          indices[f]
         }
-        scripts.collect{|s| FileUtils.rm(s.build_to)}
       end
     end
 
-    def compress_scripts
-      compressor = YUI::JavaScriptCompressor.new(:munge => false)
-      compress(".js", "#{@build_to}/scripts.js", compressor)
+    def files_for_all_products
+      return @files_for_all_products if @files_for_all_products
+      SlinkyError.batch_errors do
+        @files_for_all_products = @config.produce.keys.map{|product|
+          files_for_product(product)
+        }.flatten.uniq
+      end
     end
 
-    def compress_styles
-      compressor = YUI::CssCompressor.new()
+    def compress_product product
+      compressor = compressor_for_product product
+      post_processor = post_processor_for_product product
 
-      compress(".css", "#{@build_to}/styles.css", compressor){|s, css|
-        css.gsub(CSS_URL_MATCHER){|url|
-          p = s.relative_output_path.dirname.to_s + "/#{$1}"
-          "url('/#{p}')"
-        }
+      s = files_for_product(product).map{|mf|
+        f = File.open(mf.build_to.to_s, 'rb'){|f| f.read}
+        post_processor ? (post_processor.call(mf, f)) : f
+      }.join("\n")
+
+      # Make the directory the product is in
+      FileUtils.mkdir_p("#{@build_to}/#{Pathname.new(product).dirname}")
+      File.open("#{@build_to}/#{product}", "w+"){|f|
+        unless @no_minify
+          f.write(compressor.compress(s))
+        else
+          f.write(s)
+        end
       }
     end
 
+    # These are special cases for simplicity and backwards
+    # compatability. If no products are defined, we have two default
+    # products, one which includes are .js files in the repo and one
+    # that includes all .css files. This method produces an HTML include
+    # string for all of the .js files.
+    def scripts_string
+      product_string ConfigReader::DEFAULT_SCRIPT_PRODUCT
+    end
+
+    # These are special cases for simplicity and backwards
+    # compatability. If no products are defined, we have two default
+    # products, one which includes are .js files in the repo and one
+    # that includes all .css files. This method produces an HTML include
+    # string for all of the .css files.
     def styles_string
+      product_string ConfigReader::DEFAULT_STYLE_PRODUCT
+    end
+
+    # Produces a string of HTML that includes all of the files for the
+    # given product.
+    def product_string product
       if @devel
-        dependency_list.reject{|x| x.output_path.extname != ".css"}.collect{|d|
-          %Q\<link rel="stylesheet" href="/#{d.relative_output_path}" />\
+        files_for_product(product).map{|f|
+          html_for_path("/#{f.relative_output_path}")
         }.join("\n")
       else
-        %Q\<link rel="stylesheet" href="/styles.css?#{rand(999999999)}" />\
+        html_for_path("#{product}?#{rand(999999999)}")
       end
     end
 
@@ -149,59 +256,32 @@ module Slinky
     # (required, by), each of which describes an edge.
     #
     # @return [[ManifestFile, ManifestFile]] the graph
-    def build_dependency_graph
+    def dependency_graph
+      return @dependency_graph if @dependency_graph
+
       graph = []
       files(false).each{|mf|
-        mf.directives[:slinky_require].each{|rf|
-          required = mf.parent.find_by_path(rf, true)
-          if required.size > 0
-            required.each{|x|
-              graph << [x, mf]
-            }
-          else
-            error = "Could not find file #{rf} required by #{mf.source}"
-            $stderr.puts error.foreground(:red)
-            raise FileNotFoundError.new(error)
-          end
-        } if mf.directives[:slinky_require]
+        mf.dependencies.each{|d|
+          graph << [d, mf]
+        }
       }
-      @dependency_graph = graph
+
+      @dependency_graph = Graph.new(files(false), graph)
     end
 
-    # Builds a list of files in topological order, so that when
-    # required in this order all dependencies are met. See
-    # http://en.wikipedia.org/wiki/Topological_sorting for more
-    # information.
     def dependency_list
-      build_dependency_graph unless @dependency_graph
-      graph = @dependency_graph.clone
-      # will contain sorted elements
-      l = []
-      # start nodes, those with no incoming edges
-      s = files(false).reject{|mf| mf.directives[:slinky_require]}
-      while s.size > 0
-        n = s.delete s.first
-        l << n
-        files(false).each{|m|
-          e = graph.find{|e| e[0] == n && e[1] == m}
-          next unless e
-          graph.delete e
-          s << m unless graph.any?{|e| e[1] == m}
-        }
-      end
-      if graph != []
-        problems = graph.collect{|e| e.collect{|x| x.source}.join(" -> ")}
-        $stderr.puts "Dependencies #{problems.join(", ")} could not be satisfied".foreground(:red)
-        raise DependencyError
-      end
-      l
+      dependency_graph.dependency_list
     end
 
     def build
       @manifest_dir.build
       unless @devel
-        compress_scripts
-        compress_styles
+        @config.produce.keys.each{|product|
+          compress_product(product)
+        }
+
+        # clean up the files that have been processed
+        files_for_all_products.each{|mf| FileUtils.rm(mf.build_to, :force => true)}
       end
     end
 
@@ -225,10 +305,46 @@ module Slinky
       end
     end
 
+    def compressor_for_product product
+      case type_for_product(product)
+      when ".js"
+        YUI::JavaScriptCompressor.new(:munge => false)
+      when ".css"
+        YUI::CssCompressor.new()
+      end
+    end
+
+    def post_processor_for_product product
+      case type_for_product(product)
+      when ".css"
+         lambda{|s, css| css.gsub(CSS_URL_MATCHER){|url|
+             p = s.relative_output_path.dirname.to_s + "/#{$1}"
+             "url('/#{p}')"
+           }}
+      end
+    end
+
     def invalidate_cache
       @files = nil
       @dependency_graph = nil
       @md5 = nil
+      @files_for_all_products = nil
+    end
+
+    def html_for_path path
+      ext = path.split("?").first.split(".").last
+      case ext
+      when "css"
+        %Q|<link rel="stylesheet" href="#{path}" />|
+      when "js"
+        %Q|<script type="text/javascript" src="#{path}"></script>|
+      else
+        raise InvalidConfigError.new("Unsupported file extension #{ext}")
+      end
+    end
+
+    def type_for_product product
+      "." + product.split(".")[-1]
     end
 
     def manifest_update paths
@@ -278,6 +394,11 @@ module Slinky
     #
     # @return [ManifestFile] the manifest file at that path if one exists
     def find_by_path path, allow_multiple = false
+      if path[0] == '/'
+        # refer absolute paths to the manifest
+        return @manifest.find_by_path(path[1..-1], allow_multiple)
+      end
+
       components = path.to_s.split(File::SEPARATOR).reject{|x| x == ""}
       case components.size
       when 0
@@ -351,9 +472,13 @@ module Slinky
       unless File.directory?(@build_dir.to_s)
         FileUtils.mkdir(@build_dir.to_s)
       end
-      (@files + @children).each{|m|
-        m.build
-      }
+
+      if (@files + @children).map {|m| m.build}.any?
+        @build_dir
+      else
+        FileUtils.rmdir(@build_dir.to_s)
+        nil
+      end
     end
 
     def to_s
@@ -383,6 +508,26 @@ module Slinky
       @last_md5 = nil
     end
 
+    # Gets the list of manifest files that this one depends on
+    # according to its directive list and the dependencies config
+    # option.
+    #
+    # Throws a FileNotFoundError if a dependency doesn't exist in the
+    # tree.
+    def dependencies
+      SlinkyError.batch_errors do
+        (@directives[:slinky_require].to_a +
+         @manifest.config.dependencies["/" + relative_source_path.to_s].to_a).map{|rf|
+          required = parent.find_by_path(rf, true).flatten
+          if required.empty?
+            error = "Could not find file #{rf} required by /#{relative_source_path}"
+            SlinkyError.raise FileNotFoundError, error
+          end
+          required
+        }.flatten
+      end
+    end
+
     # Predicate which determines whether the supplied name is the same
     # as the file's name, taking into account compiled file
     # extensions. For example, if mf refers to "/tmp/test/hello.sass",
@@ -415,6 +560,16 @@ module Slinky
       end
     end
 
+    # Predicate which determines whether the file matches (see
+    # `ManifestFile#matches?`) the full path relative to the manifest
+    # root.
+    def matches_path? s, match_glob = false
+      p = Pathname.new(s)
+      dir = Pathname.new("/" + relative_source_path.to_s).dirname
+      matches?(p.basename.to_s, match_glob) &&
+         dir == p.dirname
+    end
+
     # Predicate which determines whether the file is the supplied path
     # or lies on supplied tree
     def in_tree? path
@@ -440,12 +595,12 @@ module Slinky
 
     # returns the source path relative to the manifest directory
     def relative_source_path
-      Pathname.new(@source).relative_path_from Pathname.new(@manifest.dir)
+      Pathname.new(@source).relative_path_from(Pathname.new(@manifest.dir))
     end
 
     # Returns the output path relative to the manifest directory
     def relative_output_path
-      output_path.relative_path_from Pathname.new(@manifest.dir)
+      output_path.relative_path_from(Pathname.new(@manifest.dir))
     end
 
     # Looks through the file for directives
@@ -482,8 +637,11 @@ module Slinky
         out = File.read(path)
         out.gsub!(DEPENDS_DIRECTIVE, "")
         out.gsub!(REQUIRE_DIRECTIVE, "")
-        out.gsub!(SCRIPTS_DIRECTIVE, @manifest.scripts_string)
-        out.gsub!(STYLES_DIRECTIVE, @manifest.styles_string)
+        out.gsub!(SCRIPTS_DIRECTIVE){ @manifest.scripts_string }
+        out.gsub!(STYLES_DIRECTIVE){ @manifest.styles_string }
+        out.gsub!(PRODUCT_DIRECTIVE){
+          @manifest.product_string($2[1..-2])
+        }
         to = to || Tempfile.new("slinky").path + ".cache"
         File.open(to, "w+"){|f|
           f.write(out)
@@ -528,27 +686,32 @@ module Slinky
         find_directives
       end
 
-      depends = @directives[:slinky_depends].map{|f|
-        p = parent.find_by_path(f, true)
-        $stderr.puts "File #{f} depended on by #{@source} not found".foreground(:red) unless p.size > 0
-        p
-      }.flatten.compact if @directives[:slinky_depends]
-      depends ||= []
-      @processing = true
-      # process each file on which we're dependent, watching out for 
-      # infinite loops
-      depends.each{|f| f.process }
-      @processing = false
-
-      # get hash of source file
-      if @last_path && hash == @last_md5 && depends.all?{|f| f.updated < start_time}
-        @last_path
-      else
-        @last_md5 = hash
-        @updated = Time.now
-        # mangle file appropriately
-        f = should_compile ? (compile @source) : @source
-        @last_path = handle_directives(f, to)
+      SlinkyError.batch_errors do
+        depends = @directives[:slinky_depends].map{|f|
+          p = parent.find_by_path(f, true)
+          unless p.size > 0        
+            SlinkyError.raise DependencyError,
+                              "File #{f} dependedon by #{@source} not found" 
+          end
+          p
+        }.flatten.compact if @directives[:slinky_depends]
+        depends ||= []
+        @processing = true
+        # process each file on which we're dependent, watching out for 
+        # infinite loops
+        depends.each{|f| f.process }
+        @processing = false
+
+        # get hash of source file
+        if @last_path && hash == @last_md5 && depends.all?{|f| f.updated < start_time}
+          @last_path
+        else
+          @last_md5 = hash
+          @updated = Time.now
+          # mangle file appropriately
+          f = should_compile ? (compile @source) : @source
+          @last_path = handle_directives(f, to)
+        end
       end
     end
 
@@ -560,25 +723,29 @@ module Slinky
     # Builds the file by handling and compiling it and then copying it
     # to the build path
     def build
+      return nil unless should_build
+      
       if !File.exists? @build_path
         FileUtils.mkdir_p(@build_path)
       end
       to = build_to
-      begin 
-        path = process to
-      rescue
-        raise BuildFailedError
-      end
+      path = process to
 
-      if !path
-        raise BuildFailedError
-      elsif path != to
+      if path != to
         FileUtils.cp(path.to_s, to.to_s)
         @last_built = Time.now
       end
       to
     end
 
+    def should_build
+      @manifest.files_for_all_products.include?(self) || ![".js", ".css"].include?(output_path.extname)
+    end
+
+    def inspect
+      to_s
+    end
+
     def to_s
       "<Slinky::ManifestFile '#{@source}'>"
     end