rake-pipeline-fork 0.8.0

data/lib/rake-pipeline/cli.rb

@@ -0,0 +1,56 @@
+require "thor"
+
+module Rake
+  class Pipeline
+    class CLI < Thor
+      class_option :assetfile, :default => "Assetfile", :aliases => "-c"
+      default_task :build
+
+      desc "build", "Build the project."
+      method_option :pretend, :type => :boolean, :aliases => "-p"
+      method_option :clean, :type => :boolean, :aliases => "-C"
+      def build
+        if options[:pretend]
+          project.output_files.each do |file|
+            say_status :create, relative_path(file)
+          end
+        else
+          options[:clean] ? project.clean : project.cleanup_tmpdir
+          project.invoke
+        end
+      end
+
+      desc "clean", "Remove the pipeline's temporary and output files."
+      method_option :pretend, :type => :boolean, :aliases => "-p"
+      def clean
+        if options[:pretend]
+          project.files_to_clean.each do |file|
+            say_status :remove, relative_path(file)
+          end
+        else
+          project.clean
+        end
+      end
+
+      desc "server", "Run the Rake::Pipeline preview server."
+      def server
+        require "rake-pipeline/server"
+        Rake::Pipeline::Server.new.start
+      end
+
+    private
+      def project
+        @project ||= Rake::Pipeline::Project.new(options[:assetfile])
+      end
+
+      # @param [FileWrapper|String] path
+      # @return [String] The path to the file with the current
+      #   directory stripped out.
+      def relative_path(path)
+        pathstr = path.respond_to?(:fullpath) ? path.fullpath : path
+        pathstr.sub(%r|#{Dir.pwd}/|, '')
+      end
+    end
+  end
+end
+

data/lib/rake-pipeline/dsl.rb

@@ -0,0 +1,9 @@
+module Rake
+  class Pipeline
+    module DSL
+    end
+  end
+end
+
+require 'rake-pipeline/dsl/pipeline_dsl'
+require 'rake-pipeline/dsl/project_dsl'

data/lib/rake-pipeline/dsl/pipeline_dsl.rb

@@ -0,0 +1,246 @@
+module Rake
+  class Pipeline
+    module DSL
+      # This class is used by {ProjectDSL} to provide a convenient DSL for
+      # configuring a pipeline.
+      #
+      # All instance methods of {PipelineDSL} are available in the context
+      # the block passed to +Rake::Pipeline.+{Pipeline.build}.
+      class PipelineDSL
+        # @return [Pipeline] the pipeline the DSL should configure
+        attr_reader :pipeline
+
+        # Configure a pipeline with a passed in block.
+        #
+        # @param [Pipeline] pipeline the pipeline that the PipelineDSL
+        #   should configure.
+        # @param [Proc] block the block describing the
+        #   configuration. This block will be evaluated in
+        #   the context of a new instance of {PipelineDSL}
+        # @return [void]
+        def self.evaluate(pipeline, options, &block)
+          dsl = new(pipeline)
+
+          # If any before filters, apply them to the pipeline.
+          # They will be run in reverse of insertion order.
+          if before_filters = options[:before_filters]
+            before_filters.each do |klass, args, block|
+              dsl.filter klass, *args, &block
+            end
+          end
+
+          # Evaluate the block in the context of the DSL.
+          dsl.instance_eval(&block)
+
+          # If any after filters, apply them to the pipeline.
+          # They will be run in insertion order.
+          if after_filters = options[:after_filters]
+            after_filters.each do |klass, args, block|
+              dsl.filter klass, *args, &block
+            end
+          end
+
+          # the FinalizingFilter should always come after all
+          # user specified after filters
+          pipeline.finalize
+        end
+
+        # Create a new {PipelineDSL} to configure a pipeline.
+        #
+        # @param [Pipeline] pipeline the pipeline that the PipelineDSL
+        #   should configure.
+        # @return [void]
+        def initialize(pipeline)
+          @pipeline = pipeline
+        end
+
+        # Add an input location and files to a pipeline.
+        #
+        # @example
+        #   !!!ruby
+        #   Rake::Pipeline::Project.build do
+        #     input "app" do
+        #       input "assets", "**/*.js"
+        #       # ...
+        #     end
+        #   end
+        #
+        # @param [String] root the root path where the pipeline
+        #   should find its input files.
+        # @param [String] glob a file pattern that represents
+        #   the list of files that the pipeline should
+        #   process within +root+. The default is +"**/*"+.
+        # @return [void]
+        def input(root, glob="**/*")
+          pipeline.add_input root, glob
+        end
+
+        # Add a filter to the pipeline.
+        #
+        # In addition to a filter class, {#filter} takes a
+        # block that describes how the filter should map
+        # input files to output files.
+        #
+        # By default, the block maps an input file into
+        # an output file with the same name.
+        #
+        # Any additional arguments passed to {#filter} will
+        # be passed on to the filter class's constructor.
+        #
+        # @see Filter#outputs Filter#output (for an example
+        #   of how a list of input files gets mapped to
+        #   its outputs)
+        #
+        # @param [Class] filter_class the class of the filter.
+        # @param [Array] ctor_args a list of arguments to pass
+        #   to the filter's constructor.
+        # @param [Proc] block an output file name generator.
+        # @return [void]
+        def filter(filter_class, *ctor_args, &block)
+          filter = filter_class.new(*ctor_args, &block)
+          pipeline.add_filter(filter)
+        end
+
+        # Apply a number of filters, but only to files matching
+        # a particular pattern.
+        #
+        # Inside the block passed to {#match match}, you may
+        # specify any number of filters that should be applied
+        # to files matching the pattern.
+        #
+        # @param [String] pattern a glob pattern to match
+        # @param [Proc] block a block that supplies filters
+        # @return [Matcher]
+        #
+        # @example
+        #   !!!ruby
+        #   Rake::Pipeline::Project.build do
+        #     output "public"
+        #
+        #     input "app/assets" do
+        #       # compile coffee files into JS files
+        #       match "*.coffee" do
+        #         coffee_script
+        #       end
+        #
+        #       # because the previous step converted coffeee
+        #       # into JS, the coffee files will be included here
+        #       match "*.js" do
+        #         uglify
+        #         concat "application.js"
+        #       end
+        #     end
+        #   end
+        def match(pattern, &block)
+          matcher = pipeline.copy(Matcher, &block)
+          matcher.glob = pattern
+          pipeline.add_filter matcher
+          matcher
+        end
+
+        # Reject files matching a pattern or block. You may specify
+        # a glob or a glob.
+        #
+        # @param [String] pattern a glob pattern to match
+        # @param [Proc] a block used to evaluate each file. Returning 
+        # true will skip that file.
+        # @return [RejectMatcher]
+        #
+        # @example
+        #   !!!ruby
+        #   Rake::Pipeline::Project.build do
+        #     output "public"
+        #
+        #     input "app/assets" do
+        #       # reject everything maching *.min
+        #       reject "*.min"
+        #     end
+        #
+        #     input "app/javascripts" do
+        #       reject do |file|
+        #         # process the file here
+        #       end
+        #     end
+        #   end
+        def reject(pattern = '', &block)
+          matcher = pipeline.copy(RejectMatcher)
+          matcher.glob = pattern
+          matcher.block = block
+          pipeline.add_filter matcher
+          matcher
+        end
+        alias_method :exclude, :reject
+        alias_method :skip, :reject
+
+        # Apply filters in a sorted fashion. Use this when you need
+        # something other than file name ordering.
+        #
+        # @param [Proc] block used to sort inputs
+        # @return [SortedPipeline]
+        #
+        # @example
+        #   !!!ruby
+        #   Rake::Pipeline::Project.build do
+        #       match "*.js" do
+        #         # inputs will be sorted according to the block and
+        #         # passed to concat
+        #         sort do |f1, f2|
+        #           # reverse the inputs
+        #           f2.fullpath <=> f1.fullpath
+        #         end
+        #         concat "application.js"
+        #       end
+        #     end
+        #   end
+        def sort(&block)
+          sorter = pipeline.copy(SortedPipeline)
+          sorter.comparator = block
+          pipeline.add_filter sorter
+          sorter
+        end
+
+        # Specify the output directory for the pipeline.
+        #
+        # @param [String] root the output directory.
+        # @return [void]
+        def output(root)
+          pipeline.output_root = root
+        end
+
+        # A helper method for adding a concat filter to
+        # the pipeline.
+        # If the first argument is an Array, it adds a new
+        # {OrderingConcatFilter}, otherwise it adds a new
+        # {ConcatFilter}.
+        #
+        # @see OrderingConcatFilter#initialize
+        # @see ConcatFilter#initialize
+        def concat(*args, &block)
+          if args.first.kind_of?(Array)
+            filter(Rake::Pipeline::OrderingConcatFilter, *args, &block)
+          else
+            filter(Rake::Pipeline::ConcatFilter, *args, &block)
+          end
+        end
+        alias_method :copy, :concat
+
+        # A helper method for adding a gsub filter to the pipeline.
+        #
+        # @see GsubFilter#initialize
+        def gsub(*args, &block)
+          filter(Rake::Pipeline::GsubFilter, *args, &block)
+        end
+        alias_method :replace, :gsub
+
+        # A helper method like gsub, but removes everything
+        # specified by the matcher. The matcher is the first argument
+        # passed to String#gsub
+        #
+        # @see String#gsub
+        def strip(matcher)
+          filter(Rake::Pipeline::GsubFilter, matcher, '')
+        end
+      end
+    end
+  end
+end

data/lib/rake-pipeline/dsl/project_dsl.rb

@@ -0,0 +1,108 @@
+module Rake
+  class Pipeline
+    module DSL
+      # This class exists purely to provide a convenient DSL for
+      # configuring a project.
+      #
+      # All instance methods of {ProjectDSL} are available in the context
+      # the block passed to +Rake::Pipeline::Project.+{Project.build}.
+      #
+      # When configuring a project, you *must* provide an output root
+      # and a series of files using at least one {#input} block.
+      class ProjectDSL
+        # @return [Project] the project the DSL should configure
+        attr_reader :project
+
+        # Configure a project with a passed in block.
+        #
+        # @param [Project] project the project that the ProjectDSL
+        #   should configure.
+        # @param [Proc] block the block describing the
+        #   configuration. This block will be evaluated in
+        #   the context of a new instance of {ProjectDSL}
+        # @return [void]
+        def self.evaluate(project, &block)
+          new(project).instance_eval(&block)
+        end
+
+        # Create a new {ProjectDSL} to configure a project.
+        #
+        # @param [Project] project
+        #   the project that the ProjectDSL should configure.
+        # @return [void]
+        def initialize(project)
+          @project = project
+          @before_filters = []
+          @after_filters = []
+          @project.before_filters = @before_filters
+          @project.after_filters = @after_filters
+        end
+
+        # Add a filter to every input block. The parameters
+        # to +before_filter+ are the same as the parameters
+        # to {PipelineDSL#filter}.
+        #
+        # Filters will be executed before the specified
+        # filters in reverse of insertion order.
+        #
+        # @see {PipelineDSL#filter}
+        def before_filter(klass, *args, &block)
+          @before_filters.unshift [klass, args, block]
+        end
+
+        # Add a filter to every input block. The parameters
+        # to +after_filter+ are the same as the parameters
+        # to {PipelineDSL#filter}.
+        #
+        # Filters will be executed after the specified
+        # filters in insertion order.
+        #
+        # @see {PipelineDSL#filter}
+        def after_filter(klass, *args, &block)
+          @after_filters.push [klass, args, block]
+        end
+
+        # Specify the default output directory for the project.
+        #
+        # Pipelines created in this project will place their
+        # outputs here unless the value is overriden in their
+        # {#input} block.
+        #
+        # @param [String] root the output directory.
+        # @return [void]
+        def output(root)
+          project.default_output_root = root
+        end
+
+        # Specify the location of the root temporary directory.
+        #
+        # Pipelines will store intermediate build artifacts
+        # in a subdirectory of this directory.
+        #
+        # This defaults to "tmp" in the current working directory.
+        #
+        # @param [String] root the temporary directory
+        # @return [void]
+        def tmpdir(root)
+          project.tmpdir = root
+        end
+
+        # Add a new pipeline with the given inputs to the project.
+        #
+        # @see Project.build_pipeline
+        def input(*inputs, &block)
+          # Allow pipelines without a specified block. This is possible
+          # if before and after filters are all that are needed for a
+          # given input.
+          block = proc {} unless block_given?
+          project.build_pipeline(*inputs, &block)
+        end
+        alias inputs input
+
+        def map(path, &block)
+          project.maps[path] = block
+        end
+      end
+    end
+  end
+end

data/lib/rake-pipeline/dynamic_file_task.rb

@@ -0,0 +1,194 @@
+module Rake
+  class Pipeline
+    # This class extends Rake's {Rake::FileTask} class to add support
+    # for dynamic dependencies. Typically, Rake handles static dependencies,
+    # where a file's dependencies are known before the task is invoked.
+    # A {DynamicFileTask} also supports dynamic dependencies, meaning the
+    # file's dependencies can be determined just before invoking the task.
+    # Because calculating a file's dependencies at runtime may be an expensive
+    # operation (it could involve reading the file from disk and parsing it
+    # to extract dependency information, for example), the results of this
+    # calculation are stored on disk in a manifest file, and reused on
+    # subsequent runs if possible.
+    #
+    # For example, consider this file app.c:
+    #
+    #     #include "app.h"
+    #     some_stuff();
+    #
+    # If we have a task that compiles app.c into app.o, it needs to
+    # process app.c to look for additional dependencies specified
+    # by the file itself.
+    class DynamicFileTask < Rake::FileTask
+      class ManifestRequired < StandardError
+        def to_s
+          "DynamicFileTask's cannot be invoked without a manifest."
+        end
+      end
+
+      attr_accessor :manifest, :last_manifest
+
+      # @return [Boolean] true if the task has a block to invoke
+      #   for dynamic dependencies, false otherwise.
+      def has_dynamic_block?
+        !!@dynamic
+      end
+
+      # @return [ManifestEntry] the manifest entry from the current
+      #   manifest. This is the entry that will be written to disk after
+      #   the task runs.
+      def manifest_entry
+        manifest[name]
+      end
+
+      # Set the current manifest entry,
+      #
+      # @param [ManifestEntry] new_entry
+      # @return [ManifestEntry]
+      def manifest_entry=(new_entry)
+        manifest[name] = new_entry
+      end
+
+      def last_manifest_entry
+        last_manifest[name]
+      end
+
+      # Invoke this task. This method only checks to see if there
+      # is a manifest then delegates to super
+      def invoke(*args)
+        raise ManifestRequired if has_dynamic_block? && !manifest
+        super
+      end
+
+      # In addition to the regular FileTask check, a DynamicFileTask 
+      # should be invoked when any of it's prerequisites are required,
+      # there is no manifest or it's dependencies are out of date.
+      #
+      # @return [Boolean]
+      def needed?
+        return true if super
+
+        return true if prerequisites_needed?
+
+        # if we have no manifest, this file task is needed
+        return true unless last_manifest_entry
+
+        # If any of this task's dynamic dependencies have changed,
+        # this file task is needed
+        last_manifest_entry.deps.each do |dep, time|
+          return true if File.mtime(dep).to_i > time
+        end
+
+        # Otherwise, it's not needed
+        false
+      end
+
+      # Add a block that will return dynamic dependencies. This
+      # block can assume that all static dependencies are up
+      # to date.
+      #
+      # @return [DynamicFileTask] self
+      def dynamic(&block)
+        @dynamic = block
+        self
+      end
+
+      # Invoke the task's dynamic block.
+      def invoke_dynamic_block
+        @dynamic.call(self)
+      end
+
+      # At runtime, we will call this to get dynamic prerequisites.
+      #
+      # @return [Array[String]] an array of paths to the task's
+      #   dynamic dependencies.
+      def dynamic_prerequisites
+        @dynamic_prerequisites ||= begin
+          dynamics = if has_dynamic_block?
+            dynamic_prerequisites_from_manifest || invoke_dynamic_block
+          else
+            []
+          end
+
+          # Make sure we don't dynamically depend on ourselves, as
+          # that will create a circular reference, and that makes
+          # everybody sad.
+          dynamics.reject { |x| x == name }
+        end
+      end
+
+      # Override rake's invoke_prerequisites method to invoke
+      # static prerequisites and then any dynamic prerequisites.
+      def invoke_prerequisites(task_args, invocation_chain)
+        super
+
+        raise ManifestRequired if has_dynamic_block? && !manifest
+
+        # Retrieve the dynamic prerequisites. If all goes well,
+        # we will not have to invoke the dynamic block to do this.
+        dynamics = dynamic_prerequisites
+
+        # invoke dynamic prerequisites just as we would invoke
+        # static prerequisites.
+        dynamics.each do |prereq|
+          task = lookup_prerequisite(prereq)
+          prereq_args = task_args.new_scope(task.arg_names)
+          task.invoke_with_call_chain(prereq_args, invocation_chain)
+        end
+
+        # Create a new manifest entry for each dynamic dependency.
+        # When the pipeline finishes, these manifest entries will be written
+        # to the file system.
+        entry = Rake::Pipeline::ManifestEntry.new
+
+        dynamics.each do |dynamic|
+          entry.deps.merge!(dynamic => mtime_or_now(dynamic).to_i)
+        end
+
+        self.manifest_entry = entry
+      end
+
+      # After invoking a task, add the mtime of the task's output
+      # to its current manifest entry.
+      def invoke_with_call_chain(*)
+        super
+
+        manifest_entry.mtime = mtime_or_now(name).to_i
+      end
+
+    private
+      # @return the mtime of the given file if it exists, and
+      #   the current time otherwise.
+      def mtime_or_now(filename)
+        File.file?(filename) ? File.mtime(filename) : Time.now
+      end
+
+      # @return [Array<String>] a list of file paths that this
+      #   task depends on.
+      # @return [nil] if the dependencies couldn't be read
+      #   from the manifest.
+      def dynamic_prerequisites_from_manifest
+        # Try to avoid invoking the dynamic block if this file
+        # is not needed. If so, we may have all the information
+        # we need in the manifest file.
+        if !needed? && last_manifest_entry
+          mtime = last_manifest_entry.mtime
+        end
+
+        # If the output file of this task still exists and
+        # it hasn't been updated, we can simply return the
+        # list of dependencies in the manifest, which
+        # come from the return value of the dynamic block
+        # in a previous run.
+        if File.exist?(name) && mtime == File.mtime(name).to_i
+          return last_manifest_entry.deps.map { |k,v| k }
+        end
+      end
+
+      def prerequisites_needed?
+        prerequisite_tasks.any? { |n| n.needed? }
+      end
+    end
+  end
+end
+