rake-pipeline 0.5.0 → 0.7.0

data/lib/rake-pipeline/dynamic_file_task.rb

@@ -0,0 +1,188 @@
+module Rake
+  class Pipeline
+    # This class extends Rake's {Rake::FileTask} class to add support
+    # for dynamic dependencies. Typically, Rake handles static dependencies,
+    # where a file's dependencies are known before the task is invoked.
+    # A {DynamicFileTask} also supports dynamic dependencies, meaning the
+    # file's dependencies can be determined just before invoking the task.
+    # Because calculating a file's dependencies at runtime may be an expensive
+    # operation (it could involve reading the file from disk and parsing it
+    # to extract dependency information, for example), the results of this
+    # calculation are stored on disk in a manifest file, and reused on
+    # subsequent runs if possible.
+    #
+    # For example, consider this file app.c:
+    #
+    #     #include "app.h"
+    #     some_stuff();
+    #
+    # If we have a task that compiles app.c into app.o, it needs to
+    # process app.c to look for additional dependencies specified
+    # by the file itself.
+    class DynamicFileTask < Rake::FileTask
+      attr_writer :last_manifest
+      attr_writer :manifest
+
+      # @return [Boolean] true if the task has a block to invoke
+      #   for dynamic dependencies, false otherwise.
+      def has_dynamic_block?
+        !!@dynamic
+      end
+
+      def last_manifest
+        @last_manifest ||= Rake::Pipeline::Manifest.new
+      end
+
+      def manifest
+        @manifest ||= Rake::Pipeline::Manifest.new
+      end
+
+      # @return [ManifestEntry] the manifest entry from the last time
+      #   this task was run, usually read off the filesystem.
+      def last_manifest_entry
+        last_manifest[name]
+      end
+
+      # @return [ManifestEntry] the manifest entry from the current
+      #   manifest. This is the entry that will be written to disk after
+      #   the task runs.
+      def manifest_entry
+        manifest[name]
+      end
+
+      # Set the current manifest entry,
+      #
+      # @param [ManifestEntry] new_entry
+      # @return [ManifestEntry]
+      def manifest_entry=(new_entry)
+        manifest[name] = new_entry
+      end
+
+      # In addition to the regular FileTask check, A DynamicFileTask is
+      # needed if it has no manifest entry from a previous run, or if
+      # one of its dynamic dependencies has been modified.
+      #
+      # @return [Boolean]
+      def needed?
+        return true if super
+
+        # if we have no manifest, this file task is needed
+        return true unless last_manifest_entry
+
+        # If any of this task's dynamic dependencies have changed,
+        # this file task is needed
+        last_manifest_entry.deps.each do |dep, time|
+          return true if File.mtime(dep) > time
+        end
+
+        # Otherwise, it's not needed
+        false
+      end
+
+      # Add a block that will return dynamic dependencies. This
+      # block can assume that all static dependencies are up
+      # to date.
+      #
+      # @return [DynamicFileTask] self
+      def dynamic(&block)
+        @dynamic = block
+        self
+      end
+
+      # Invoke the task's dynamic block.
+      def invoke_dynamic_block
+        @dynamic.call(self)
+      end
+
+      # At runtime, we will call this to get dynamic prerequisites.
+      #
+      # @return [Array[String]] an array of paths to the task's
+      #   dynamic dependencies.
+      def dynamic_prerequisites
+        @dynamic_prerequisites ||= begin
+          dynamics = if has_dynamic_block?
+            dynamic_prerequisites_from_manifest || invoke_dynamic_block
+          else
+            []
+          end
+
+          # Make sure we don't dynamically depend on ourselves, as
+          # that will create a circular reference, and that makes
+          # everybody sad.
+          dynamics.reject { |x| x == name }
+        end
+      end
+
+      # Override rake's invoke_prerequisites method to invoke
+      # static prerequisites and then any dynamic prerequisites.
+      def invoke_prerequisites(task_args, invocation_chain)
+        super
+
+        # If we don't have a dynamic block, just act like a regular FileTask.
+        return unless has_dynamic_block?
+
+        # Retrieve the dynamic prerequisites. If all goes well,
+        # we will not have to invoke the dynamic block to do this.
+        dynamics = dynamic_prerequisites
+
+        # invoke dynamic prerequisites just as we would invoke
+        # static prerequisites.
+        dynamics.each do |prereq|
+          task = lookup_prerequisite(prereq)
+          prereq_args = task_args.new_scope(task.arg_names)
+          task.invoke_with_call_chain(prereq_args, invocation_chain)
+        end
+
+        # Create a new manifest entry for each dynamic dependency.
+        # When the pipeline finishes, these manifest entries will be written
+        # to the file system.
+        entry = Rake::Pipeline::ManifestEntry.new
+
+        dynamics.each do |dynamic|
+          entry.deps.merge!(dynamic => mtime_or_now(dynamic))
+        end
+
+        self.manifest_entry = entry
+      end
+
+      # After invoking a task, add the mtime of the task's output
+      # to its current manifest entry.
+      def invoke_with_call_chain(*)
+        super
+        return unless has_dynamic_block?
+
+        manifest_entry.mtime = mtime_or_now(name)
+      end
+
+    private
+      # @return the mtime of the given file if it exists, and
+      #   the current time otherwise.
+      def mtime_or_now(filename)
+        File.file?(filename) ? File.mtime(filename) : Time.now
+      end
+
+      # @return [Array<String>] a list of file paths that this
+      #   task depends on.
+      # @return [nil] if the dependencies couldn't be read
+      #   from the manifest.
+      def dynamic_prerequisites_from_manifest
+        # Try to avoid invoking the dynamic block if this file
+        # is not needed. If so, we may have all the information
+        # we need in the manifest file.
+        if !needed? && last_manifest_entry
+          mtime = last_manifest_entry.mtime
+        end
+
+        # If the output file of this task still exists and
+        # it hasn't been updated, we can simply return the
+        # list of dependencies in the manifest, which
+        # come from the return value of the dynamic block
+        # in a previous run.
+        if File.exist?(name) && mtime == File.mtime(name)
+          return last_manifest_entry.deps.map { |k,v| k }
+        end
+      end
+    end
+  end
+end
+

data/lib/rake-pipeline/file_wrapper.rb

@@ -66,7 +66,7 @@ module Rake
       #
       # @return [String] the {FileWrapper}'s full path
       def fullpath
-        raise "#{root}, #{path}" unless root =~ /^\//
+        raise "#{root}, #{path}" unless root =~ /^(\/|[a-zA-Z]:[\\\/])/
         File.join(root, path)
       end
 

data/lib/rake-pipeline/filter.rb

@@ -46,7 +46,9 @@ module Rake
       attr_accessor :input_files
 
       # @return [Proc] a block that returns the relative output
-      #   filename for a particular input file.
+      #   filename for a particular input file. If the block accepts
+      #   just one argument, it will be passed the input's path. If
+      #   it accepts two, it will also be passed the input itself.
       attr_accessor :output_name_generator
 
       # @return [String] the root directory to write output files
@@ -66,6 +68,10 @@ module Rake
       #   filter should define new rake tasks on.
       attr_writer :rake_application
 
+      # @return [Rake::Pipeline] the Rake::Pipeline that contains
+      #   this filter.
+      attr_accessor :pipeline
+
       attr_writer :file_wrapper_class
 
       # @param [Proc] block a block to use as the Filter's
@@ -73,6 +79,7 @@ module Rake
       def initialize(&block)
         block ||= proc { |input| input }
         @output_name_generator = block
+        @input_files = []
       end
 
       # Invoke this method in a subclass of Filter to declare that
@@ -148,10 +155,10 @@ module Rake
         hash = {}
 
         input_files.each do |file|
-          output = output_wrapper(output_name_generator.call(file.path))
-
-          hash[output] ||= []
-          hash[output] << file
+          output_wrappers(file).each do |output|
+            hash[output] ||= []
+            hash[output] << file
+          end
         end
 
         hash
@@ -163,9 +170,7 @@ module Rake
       # @see #outputs
       # @return [Array<FileWrapper>]
       def output_files
-        input_files.inject([]) do |array, file|
-          array |= [output_wrapper(output_name_generator.call(file.path))]
-        end
+        input_files.map { |file| output_wrappers(file) }.flatten.uniq
       end
 
       # The Rake::Application that the filter should define new tasks on.
@@ -175,17 +180,27 @@ module Rake
         @rake_application || Rake.application
       end
 
+      # @param [FileWrapper] file wrapper to get paths for
+      # @return [Array<String>] array of file paths within additional dependencies
+      def additional_dependencies(input)
+        []
+      end
+
       # Generate the Rake tasks for the output files of this filter.
       #
       # @see #outputs #outputs (for information on how the output files are determined)
       # @return [void]
       def generate_rake_tasks
         @rake_tasks = outputs.map do |output, inputs|
-          dependencies = inputs.map(&:fullpath)
-
-          dependencies.each { |path| create_file_task(path) }
+          additional_paths = []
+          inputs.each do |input|
+            create_file_task(input.fullpath).dynamic do
+              additional_paths += additional_dependencies(input)
+            end
+          end
+          additional_paths.each { |path| create_file_task(path) }
 
-          create_file_task(output.fullpath, dependencies) do
+          create_file_task(output.fullpath, inputs.map(&:fullpath)) do
             output.create { generate_output(inputs, output) }
           end
         end
@@ -198,11 +213,26 @@ module Rake
       end
 
       def create_file_task(output, deps=[], &block)
-        rake_application.define_task(Rake::FileTask, output => deps, &block)
+        task = rake_application.define_task(Rake::Pipeline::DynamicFileTask, output => deps, &block)
+
+        if pipeline && pipeline.project
+          task.last_manifest = pipeline.project.last_manifest
+          task.manifest = pipeline.project.manifest
+        end
+
+        task
+      end
+
+      def output_wrappers(input)
+        output_paths(input).map do |path|
+          file_wrapper_class.new(output_root, path, encoding)
+        end
       end
 
-      def output_wrapper(file)
-        file_wrapper_class.new(output_root, file, encoding)
+      def output_paths(input)
+        args = [ input.path ]
+        args << input if output_name_generator.arity == 2
+        Array(output_name_generator.call(*args))
       end
     end
   end

data/lib/rake-pipeline/filters.rb

@@ -1 +1,3 @@
-require "rake-pipeline/filters/concat"
+require "rake-pipeline/filters/concat_filter"
+require "rake-pipeline/filters/ordering_concat_filter"
+require "rake-pipeline/filters/pipeline_finalizing_filter"

data/lib/rake-pipeline/filters/ordering_concat_filter.rb

@@ -0,0 +1,38 @@
+class Rake::Pipeline
+  # A filter that concats files in a specified order.
+  #
+  # @example
+  #   !!!ruby
+  #   Rake::Pipeline.build do
+  #     input "app/assets", "**/*.js"
+  #     output "public"
+  #
+  #     # Concat each file into libs.js but make sure
+  #     # that jQuery and Ember come first.
+  #     filter Rake::Pipeline::OrderingConcatFilter, ["jquery.js", "ember.js"], "libs.js"
+  #   end
+  class OrderingConcatFilter < ConcatFilter
+
+    # @param [Array<String>] ordering an Array of Strings
+    #   of file names that should come in the specified order
+    # @param [String] string the name of the output file to
+    #   concatenate inputs to.
+    # @param [Proc] block a block to use as the Filter's
+    #   {#output_name_generator}.
+    def initialize(ordering, string=nil, &block)
+      @ordering = ordering
+      super(string, &block)
+    end
+
+    # Extend the {#generate_output} method supplied by {ConcatFilter}.
+    # Re-orders the inputs such that the specified files come first.
+    # If a file is not in the list it will come after the specified files.
+    def generate_output(inputs, output)
+      @ordering.reverse.each do |name|
+        file = inputs.find{|i| i.path == name }
+        inputs.unshift(inputs.delete(file)) if file
+      end
+      super
+    end
+  end
+end

data/lib/rake-pipeline/filters/pipeline_finalizing_filter.rb

@@ -0,0 +1,19 @@
+module Rake
+  class Pipeline
+    # @private
+    #
+    # A built-in filter that copies a pipeline's generated files over
+    # to its output.
+    class PipelineFinalizingFilter < ConcatFilter
+
+      # @return [Array[FileWrapper]] a list of the pipeline's
+      # output files, excluding any files that were originally
+      # inputs to the pipeline, meaning they weren't processed
+      # by any filter and should not be copied to the output.
+      def input_files
+        pipeline_input_files = pipeline.input_files
+        super.reject { |file| pipeline_input_files.include?(file) }
+      end
+    end
+  end
+end

data/lib/rake-pipeline/graph.rb

@@ -0,0 +1,178 @@
+require "set"
+
+module Rake
+  class Pipeline
+    # The goal of this class is to make is easy to implement dynamic
+    # dependencies in additional_dependencies without having to parse
+    # all the files all of the time.
+    #
+    # To illustrate, imagine that we have two source files with the
+    # following inline dependencies:
+    #
+    # * application.scss
+    #   * _core.scss
+    # * admin.scss
+    #   * _admin.scss
+    #
+    # And further imagine that `_admin.scss` has an inline dependency
+    # on `_core.scss`.
+    #
+    # On initial build, we will scan all of the source files, find
+    # the dependencies, and build a node for each file, annotating
+    # the source files with `:source => true`. We also store off the
+    # `mtime` of each file in its node. We link each file to its
+    # dependencies.
+    #
+    # The `additional_dependencies` are a map of the files to their
+    # children, which will be used when generating rake tasks.
+    #
+    # Later, let's say that we change `_admin.scss`. We will need
+    # to unlink its dependencies first (on `_core.scss`), rescan
+    # the file, and create nodes for its dependencies. If no new
+    # dependencies
+
+    class Graph
+      class MissingNode < StandardError
+      end
+
+      class Node
+        # @return [String] the identifier of the node
+        attr_reader :name
+
+        # @return [Set] a Set of parent nodes
+        attr_reader :parents
+
+        # @return [Set] a Set of child nodes
+        attr_reader :children
+
+        # @return [Hash] a Hash of metadata
+        attr_reader :metadata
+
+        # @param [String] name the identifier of the node
+        # @param [Hash] metadata an optional hash of metadata
+        def initialize(name, metadata={})
+          @name = name
+          @parents = Set.new
+          @children = Set.new
+          @metadata = metadata
+        end
+
+        # A node is equal another node if it has the same name.
+        # This is because the Graph ensures that only one node
+        # with a given name can be created.
+        #
+        # @param [Node] other the node to compare
+        def ==(other)
+          @name == other.name
+        end
+      end
+
+      def initialize
+        @map = {}
+      end
+
+      # @return [Array] an Array of all of the nodes in the graph
+      def nodes
+        @map.values
+      end
+
+      # Add a new node to the graph. If an existing node with the
+      # current name already exists, do not add the node.
+      #
+      # @param [String] name an identifier for the node.
+      # @param [Hash] metadata optional metadata for the node
+      def add(name, metadata={})
+        return if @map.include?(name)
+        @map[name] = Node.new(name, metadata)
+      end
+
+      # Remove a node from the graph. Unlink its parent and children
+      # from it.
+      #
+      # If the existing node does not exist, raise.
+      #
+      # @param [String] name an identifier for the node
+      def remove(name)
+        node = verify(name)
+
+        node.parents.each do |parent_node|
+          parent_node.children.delete node
+        end
+
+        node.children.each do |child_node|
+          child_node.parents.delete node
+        end
+
+        @map.delete(name)
+      end
+
+      # Add a link from the parent to the child. This link is a
+      # two-way link, so the child will be added to the parent's
+      # `children` and the parent will be added to the child's
+      # `parents`.
+      #
+      # The parent and child are referenced by node identifier.
+      #
+      # @param [String] parent the identifier of the parent
+      # @param [String] child the identifier of the child
+      def link(parent, child)
+        parent, child = lookup(parent, child)
+
+        parent.children << child
+        child.parents << parent
+      end
+
+      # Remove a link from the parent to the child.
+      #
+      # The parent and child are referenced by node identifier.
+      #
+      # @param [String] parent the identifier of the parent
+      # @param [String] child the identifier of the child
+      def unlink(parent, child)
+        parent, child = lookup(parent, child)
+
+        parent.children.delete(child)
+        child.parents.delete(parent)
+      end
+
+      # Look up a node by name
+      #
+      # @param [String] name the identifier of the node
+      # @return [Node] the node referenced by the specified identifier
+      def [](name)
+        @map[name]
+      end
+
+    private
+      # Verify that the parent and child nodes exist, and return
+      # the nodes with the specified identifiers.
+      #
+      # The parent and child are referenced by node identifier.
+      #
+      # @param [String] parent the identifier of the parent
+      # @param [String] child the identifier of the child
+      # @return [Array(Node, Node)] the parent and child nodes
+      def lookup(parent, child)
+        parent = verify(parent)
+        child = verify(child)
+
+        return parent, child
+      end
+
+      # Verify that a node with a given identifier exists, and
+      # if it does, return it.
+      #
+      # If it does not, raise an exception.
+      #
+      # @param [String] name the identifier of the node
+      # @raise [MissingNode] if a node with the given name is
+      #   not found, raise.
+      # @return [Node] the n
+      def verify(name)
+        node = @map[name]
+        raise MissingNode, "Node #{name} does not exist" unless node
+        node
+      end
+    end
+  end
+end