rake-pipeline 0.5.0 → 0.7.0

data/.travis.yml

@@ -0,0 +1,12 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - ree
+  - jruby
+  - rbx
+
+notifications:
+  email:
+    - wycats@gmail.com
+    - dudley@steambone.org

data/Gemfile

@@ -8,3 +8,4 @@ gem "flog"
 gem "simplecov", :require => false
 gem "yard"
 gem "rdiscount"
+gem "pry"

data/README.markdown

@@ -1,4 +1,4 @@
 # Rake::Pipeline
 
 The canonical documentation for Rake::Pipeline is hosted at
-<a href="http://rubydoc.info/github/livingsocial/rake-pipeline/master/file/README.yard">rubydoc.info</a>
+<a href="http://rubydoc.info/github/livingsocial/rake-pipeline/master/file/README.yard">rubydoc.info</a>.

data/README.yard

@@ -6,45 +6,72 @@ output files based on input changes.
 
 = Usage
 
-The easiest way to use Rake::Pipeline is via a +Assetfile+ file in the
+The easiest way to use Rake::Pipeline is via an +Assetfile+ file in the
 root of your project.
 
 A sample +Assetfile+ looks like this:
 
   !!!ruby
-  input "assets"
   output "public"
 
-  # this block will take all JS inputs, wrap them in a closure,
-  # add some additional metadata, and concatenate them all into
-  # application.scripts.js.
-  match "*.js" do
-    filter ClosureWrapper
-    filter DataWrapper
-    filter Rake::Pipeline::ConcatFilter, "application.scripts.js"
-  end
+  input "assets" do
+    # this block will take all JS inputs, wrap them in a closure,
+    # add some additional metadata, and concatenate them all into
+    # application.scripts.js.
+    match "*.js" do
+      filter ClosureWrapper
+      filter DataWrapper
+      concat "application.scripts.js"
+    end
 
-  # this block will take all HTML and CSS inputs, convert them
-  # into JavaScript
-  match "*/*.{html,css}" do
-    filter DataWrapper
-    filter Rake::Pipeline::ConcatFilter, "application.assets.js"
-  end
+    # this block will take all HTML and CSS inputs, convert them
+    # into JavaScript
+    match "*/*.{html,css}" do
+      filter DataWrapper
+      concat "application.assets.js"
+    end
 
-  match "*.js" do
-    filter Rake::Pipeline::ConcatFilter, "application.js"
+    match "*.js" do
+      concat "application.js"
+    end
   end
 
-  # copy any unprocessed files over to the output directory
-  filter Rake::Pipeline::ConcatFilter
+Each +input+ block defines a collection of files, and a pipeline
+that transforms those files. Within each pipeline, you can specify
+a series of filters to describe the transformations you'd like to
+apply to the files.
+
+= Upgrading from Previous Versions
+
+The +Assetfile+ syntax has changed in version 0.6.0. In previous
+versions, each +Assetfile+ defined a single pipeline, and +input+
+statements would add input files to that pipeline. After version
+0.6.0, multiple pipelines can be defined in an +Assetfile+. The
++input+ method now takes a block, and this block defines a pipeline.
+This means that any +match+ blocks or filters must be defined
+inside an +input+ block, and no longer at the top level. For example,
+this:
+
+    !!!ruby
+    # Prior to 0.6.0
+    output "public"
+    input "assets"
+
+    match "**/*.js" do
+      concat
+    end
 
-The available options are:
+would now be written as:
 
-* {Rake::Pipeline::DSL#input input}: the directory containing your input files
-* {Rake::Pipeline::DSL#output output}: the directory to place your output files
-  like to process
-  * if you do not specify a block, the files will be
-    copied over directly.
+    !!!ruby
+    # After 0.6.0
+    output "public"
+
+    input "assets" do
+      match "**/*.js" do
+        concat
+      end
+    end
 
 = Filters
 
@@ -78,7 +105,7 @@ If you had a series of input files like:
 * +app/javascripts/three.js+
 
 and you specified the +ConcatFilter+ in your
-+AssetFile+ like:
++Assetfile+ like:
 
   !!!ruby
   filter ConcatFilter, "application.js"
@@ -109,15 +136,17 @@ This will stop `Rake::Pipeline` from trying to interpret the
 input files as `UTF-8`, which obviously will not work on
 binary data.
 
-= Built-In Filters
+= Filters
 
-At the current time, +Rake::Pipeline+ comes with a single built-in
-filter: {Rake::Pipeline::ConcatFilter}. Its implementation is
-the same as the `ConcatFilter` shown above.
++Rake::Pipeline+ comes with a built-in filter,
+{Rake::Pipeline::ConcatFilter}. Its implementation is the same as the
++ConcatFilter+ above. Other filters that are useful for web development
+like a +CoffeeScriptFilter+ and +SassFilter+ are available in
+[rake-pipeline-web-filters](https://github.com/wycats/rake-pipeline-web-filters).
 
 = Preview Server
 
-To start up the preview server, run +rakep+. This will start up
+To start up the preview server, run +rakep server+. This will start up
 a server that automatically recompiles files for you on the fly
 and serves up the files you need.
 

data/Rakefile

@@ -3,10 +3,19 @@ require "bundler/gem_tasks"
 
 directory "doc"
 
+desc "generate documentation"
 task :docs => Dir["lib/**"] do
   sh "devbin/yard doc --readme README.yard --hide-void-return"
 end
 
+desc "generate a dependency graph from the documentation"
 task :graph => ["doc", :docs] do
   sh "devbin/yard graph --dependencies | dot -Tpng -o doc/arch.png"
 end
+
+desc "run the specs"
+task :spec do
+  sh "rspec -cfs spec"
+end
+
+task :default => :spec

data/bin/rakep

@@ -1,27 +1,4 @@
 #!/usr/bin/env ruby
 
 require "rake-pipeline"
-require "rake-pipeline/middleware"
-require "rack/server"
-
-module Rake
-  class Pipeline
-    class Server < Rack::Server
-      def app
-        not_found = proc { [404, { "Content-Type" => "text/plain" }, ["not found"]] }
-        config = "Assetfile"
-
-        Middleware.new(not_found, config)
-      end
-    end
-  end
-end
-
-if ARGV[0] == "build"
-  config = "Assetfile"
-  pipeline_source = File.read(config)
-  pipeline = Rake::Pipeline.class_eval "build do\n#{pipeline_source}\nend", config, 1
-  pipeline.invoke
-else
-  Rake::Pipeline::Server.new.start
-end
+Rake::Pipeline::CLI.start

data/lib/generators/rake/pipeline/install/install_generator.rb

@@ -0,0 +1,70 @@
+module Rake
+  class Pipeline
+    class InstallGenerator < Rails::Generators::Base
+
+      desc "Install Rake::Pipeline in this Rails app"
+
+      def disable_asset_pipeline_railtie
+        say_status :config, "Updating configuration to remove asset pipeline"
+        gsub_file app, "require 'rails/all'", <<-RUBY.strip_heredoc
+          # Pick the frameworks you want:
+          require "active_record/railtie"
+          require "action_controller/railtie"
+          require "action_mailer/railtie"
+          require "active_resource/railtie"
+          require "rails/test_unit/railtie"
+        RUBY
+      end
+
+      # TODO: Support sprockets API
+      def disable_asset_pipeline_config
+        regex = /^\n?\s*#.*\n\s*(#\s*)?config\.assets.*\n/
+        gsub_file app, regex, ''
+        gsub_file Rails.root.join("config/environments/development.rb"), regex, ''
+        gsub_file Rails.root.join("config/environments/production.rb"), regex, ''
+      end
+
+      def remove_assets_group
+        regex = /^\n(#.*\n)+group :assets.*\n(.*\n)*?end\n/
+
+        gsub_file "Gemfile", regex, ''
+      end
+
+      def enable_assets_in_development
+        gsub_file "config/environments/development.rb", /^end/, "\n  config.rake_pipeline_enabled = true\nend"
+      end
+
+      # TODO: Support asset-pipeline like API
+      def add_assetfile
+        create_file "Assetfile", <<-RUBY.strip_heredoc
+          # NOTE: The Assetfile will eventually be replaced with an asset-pipeline
+          # compatible API. This is mostly important so that plugins can easily
+          # inject into the pipeline.
+          #
+          # Depending on demand and how the API shakes out, we may retain the
+          # Assetfile API but pull in the information from the Rails API.
+
+          input "app/assets"
+          output "public"
+
+          match "*.js" do
+            concat "application.js"
+          end
+
+          match "*.css" do
+            concat "application.css"
+          end
+
+          # copy any remaining files
+          concat
+        RUBY
+      end
+
+    private
+      def app
+        @app ||= Rails.root.join("config/application.rb")
+      end
+    end
+  end
+end
+

data/lib/rake-pipeline.rb

@@ -1,9 +1,15 @@
 require "rake-pipeline/file_wrapper"
 require "rake-pipeline/filter"
+require "rake-pipeline/manifest_entry"
+require "rake-pipeline/manifest"
+require "rake-pipeline/dynamic_file_task"
 require "rake-pipeline/filters"
 require "rake-pipeline/dsl"
 require "rake-pipeline/matcher"
 require "rake-pipeline/error"
+require "rake-pipeline/project"
+require "rake-pipeline/cli"
+require "rake-pipeline/graph"
 
 if defined?(Rails::Railtie)
   require "rake-pipeline/railtie"
@@ -77,7 +83,7 @@ module Rake
   #       # first, wrap all JS files in a custom filter
   #       filter ClosureFilter
   #       # then, concatenate all JS files into a single file
-  #       filter Rake::Pipeline::ConcatFilter, "application.js"
+  #       concat "application.js"
   #     end
   #
   #     # specify filters for css and scss files
@@ -89,7 +95,7 @@ module Rake
   #         input.sub(/\.scss$/, 'css')
   #       end
   #       # then, concatenate all CSS files into a single file
-  #       filter Rake::Pipeline::ConcatFilter, "application.css"
+  #       concat "application.css"
   #     end
   #
   #     # the remaining files not specified by a matcher (the
@@ -100,17 +106,15 @@ module Rake
   #     # up until this point, as well as the HTML files.
   #   end
   class Pipeline
-    # @return [String] a glob representing the input files
-    attr_accessor :input_glob
-
-    # @return [String] the directory path for the input files.
-    attr_reader   :input_root
+    # @return [Hash[String, String]] the directory paths for the input files
+    #   and their matching globs.
+    attr_accessor :inputs
 
     # @return [String] the directory path for the output files.
     attr_reader   :output_root
 
-    # @return [String] the directory path for temporary files.
-    attr_reader   :tmpdir
+    # @return [String] the directory path for temporary files
+    attr_accessor :tmpdir
 
     # @return [Array] an Array of Rake::Task objects. This
     #   property is populated by the #generate_rake_tasks
@@ -126,10 +130,33 @@ module Rake
 
     attr_writer :input_files
 
-    def initialize
-      @filters = []
-      @tmpdir = "tmp"
-      @mutex = Mutex.new
+    # @return [Project] the Project that created this pipeline
+    attr_reader :project
+
+    # @param [Hash] options
+    # @option options [Hash] :inputs
+    #   set the pipeline's {#inputs}.
+    # @option options [String] :tmpdir
+    #   set the pipeline's {#tmpdir}.
+    # @option options [String] :output_root
+    #   set the pipeline's {#output_root}.
+    # @option options [Rake::Application] :rake_application
+    #   set the pipeline's {#rake_application}.
+    def initialize(options={})
+      @filters         = []
+      @invoke_mutex    = Mutex.new
+      @clean_mutex     = Mutex.new
+      @inputs          = options[:inputs] || {}
+      @tmpdir          = options[:tmpdir] || "tmp"
+      @project         = options[:project]
+
+      if options[:output_root]
+        self.output_root = options[:output_root]
+      end
+
+      if options[:rake_application]
+        self.rake_application = options[:rake_application]
+      end
     end
 
     # Build a new pipeline taking a block. The block will
@@ -142,7 +169,7 @@ module Rake
     #     input "app/assets"
     #     output "public"
     #
-    #     filter Rake::Pipeline::ConcatFilter, "app.js"
+    #     concat "app.js"
     #   end
     #
     # @see DSL the Rake::Pipeline::DSL documentation.
@@ -150,10 +177,21 @@ module Rake
     #   the build block.
     #
     # @return [Rake::Pipeline] the newly configured pipeline
-    def self.build(&block)
-      pipeline = new
-      DSL.evaluate(pipeline, &block) if block
-      pipeline
+    def self.build(options={}, &block)
+      pipeline = new(options)
+      pipeline.build(options, &block)
+    end
+
+    # Evaluate a block using the Rake::Pipeline DSL against an
+    # existing pipeline.
+    #
+    # @see Rake::Pipeline.build
+    #
+    # @return [Rake::Pipeline] this pipeline with any modifications
+    #   made by the given block.
+    def build(options={}, &block)
+      DSL::PipelineDSL.evaluate(self, options, &block) if block
+      self
     end
 
     @@tmp_id = 0
@@ -168,19 +206,12 @@ module Rake
     # @api private
     def copy(target_class=self.class, &block)
       pipeline = target_class.build(&block)
-      pipeline.input_root = input_root
+      pipeline.inputs = inputs
       pipeline.tmpdir = tmpdir
       pipeline.rake_application = rake_application
       pipeline
     end
 
-    # Set the input root of this pipeline and expand its path.
-    #
-    # @param [String] root this pipeline's input root
-    def input_root=(root)
-      @input_root = File.expand_path(root)
-    end
-
     # Set the output root of this pipeline and expand its path.
     #
     # @param [String] root this pipeline's output root
@@ -195,7 +226,18 @@ module Rake
       @tmpdir = File.expand_path(dir)
     end
 
-    # If you specify a glob for #input_glob, this method will
+    # Add an input directory, optionally filtering which files within
+    # the input directory are included.
+    #
+    # @param [String] root the input root directory; required
+    # @param [String] pattern a pattern to match within +root+;
+    #   optional; defaults to "**/*"
+    def add_input(root, pattern = nil)
+      pattern ||= "**/*"
+      @inputs[root] = pattern
+    end
+
+    # If you specify #inputs, this method will
     # calculate the input files for the directory. If you supply
     # input_files directly, this method will simply return the
     # input_files you supplied.
@@ -207,13 +249,19 @@ module Rake
 
       assert_input_provided
 
-      expanded_root = File.expand_path(input_root)
-      files = Dir[File.join(expanded_root, input_glob)].select { |f| File.file?(f) }
+      result = []
 
-      files.map do |file|
-        relative_path = file.sub(%r{^#{Regexp.escape(expanded_root)}/}, '')
-        FileWrapper.new(expanded_root, relative_path)
+      @inputs.each do |root, glob|
+        expanded_root = File.expand_path(root)
+        files = Dir[File.join(expanded_root, glob)].sort.select { |f| File.file?(f) }
+
+        files.each do |file|
+          relative_path = file.sub(%r{^#{Regexp.escape(expanded_root)}/}, '')
+          result << FileWrapper.new(expanded_root, relative_path)
+        end
       end
+
+      result.sort
     end
 
     # for Pipelines, this is every file, but it may be overridden
@@ -240,7 +288,10 @@ module Rake
     # @param [Array<Filter>] filters a list of filters
     # @return [void]
     def add_filters(*filters)
-      filters.each { |filter| filter.rake_application = rake_application }
+      filters.each do |filter|
+        filter.rake_application = rake_application
+        filter.pipeline = self
+      end
       @filters.concat(filters)
     end
     alias add_filter add_filters
@@ -251,7 +302,7 @@ module Rake
     #
     # @return [void]
     def invoke
-      @mutex.synchronize do
+      @invoke_mutex.synchronize do
         self.rake_application = Rake::Application.new unless @rake_application
 
         setup
@@ -266,8 +317,10 @@ module Rake
     #
     # @return [void]
     def invoke_clean
-      @rake_tasks = @rake_application = nil
-      invoke
+      @clean_mutex.synchronize do
+        @rake_tasks = @rake_application = nil
+        invoke
+      end
     end
 
     # Set up the filters and generate rake tasks. In general, this method
@@ -280,22 +333,6 @@ module Rake
       generate_rake_tasks
     end
 
-    # A list of the output files that invoking this pipeline will
-    # generate.
-    #
-    # @return [Array<FileWrapper>]
-    def output_files
-      @filters.last.output_files unless @filters.empty?
-    end
-
-  protected
-    # Generate a new temporary directory name.
-    #
-    # @return [String] a unique temporary directory name
-    def self.generate_tmpname
-      "rake-pipeline-tmp-#{@@tmp_id += 1}"
-    end
-
     # Set up the filters. This will loop through all of the filters for
     # the current pipeline and wire up their input_files and output_files.
     #
@@ -303,6 +340,7 @@ module Rake
     # set up as part of this process.
     #
     # @return [void]
+    # @api private
     def setup_filters
       last = @filters.last
 
@@ -327,6 +365,31 @@ module Rake
       end
     end
 
+    # A list of the output files that invoking this pipeline will
+    # generate.
+    #
+    # @return [Array<FileWrapper>]
+    def output_files
+      @filters.last.output_files unless @filters.empty?
+    end
+
+    # Add a final filter to the pipeline that will copy the
+    # pipeline's generated files to the output.
+    #
+    # @return [void]
+    # @api private
+    def finalize
+      add_filter(Rake::Pipeline::PipelineFinalizingFilter.new)
+    end
+
+  protected
+    # Generate a new temporary directory name.
+    #
+    # @return [String] a unique temporary directory name
+    def self.generate_tmpname
+      "rake-pipeline-tmp-#{@@tmp_id += 1}"
+    end
+
     # Generate a new temporary directory name under the main tmpdir.
     #
     # @return [void]
@@ -356,10 +419,11 @@ module Rake
     # @raise Rake::Pipeline::Error if input root or glob were missing.
     # @return [void]
     def assert_input_provided
-      if !input_root || !input_glob
+      if inputs.empty?
         raise Rake::Pipeline::Error, "You cannot get input files without " \
                                      "first providing input files and an input root"
       end
     end
+
   end
 end