rake-pipeline 0.5.0

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbx
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+devbin

data/.rspec

@@ -0,0 +1 @@
+-cfs -r spec_helper.rb

data/.yardopts

@@ -0,0 +1,2 @@
+--readme README.yard
+

data/Gemfile

@@ -0,0 +1,10 @@
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in rake-pipeline.gemspec
+gemspec
+gem "flay"
+gem "flog"
+
+gem "simplecov", :require => false
+gem "yard"
+gem "rdiscount"

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2011 by LivingSocial, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

data/README.markdown

@@ -0,0 +1,4 @@
+# Rake::Pipeline
+
+The canonical documentation for Rake::Pipeline is hosted at
+<a href="http://rubydoc.info/github/livingsocial/rake-pipeline/master/file/README.yard">rubydoc.info</a>

data/README.yard

@@ -0,0 +1,149 @@
+= Rake::Pipeline
+
+Rake::Pipeline is a system for packaging assets for deployment to the
+web. It uses Rake under the hood for dependency management and updating
+output files based on input changes.
+
+= Usage
+
+The easiest way to use Rake::Pipeline is via a +Assetfile+ file in the
+root of your project.
+
+A sample +Assetfile+ looks like this:
+
+  !!!ruby
+  input "assets"
+  output "public"
+
+  # this block will take all JS inputs, wrap them in a closure,
+  # add some additional metadata, and concatenate them all into
+  # application.scripts.js.
+  match "*.js" do
+    filter ClosureWrapper
+    filter DataWrapper
+    filter Rake::Pipeline::ConcatFilter, "application.scripts.js"
+  end
+
+  # this block will take all HTML and CSS inputs, convert them
+  # into JavaScript
+  match "*/*.{html,css}" do
+    filter DataWrapper
+    filter Rake::Pipeline::ConcatFilter, "application.assets.js"
+  end
+
+  match "*.js" do
+    filter Rake::Pipeline::ConcatFilter, "application.js"
+  end
+
+  # copy any unprocessed files over to the output directory
+  filter Rake::Pipeline::ConcatFilter
+
+The available options are:
+
+* {Rake::Pipeline::DSL#input input}: the directory containing your input files
+* {Rake::Pipeline::DSL#output output}: the directory to place your output files
+  like to process
+  * if you do not specify a block, the files will be
+    copied over directly.
+
+= Filters
+
+A filter is a simple class that inherits from
+{Rake::Pipeline::Filter}. A filter must implement a single
+method, called +generate_output+, which takes
+two parameters: a list of input files and the output file.
+
+Both the input and output files are {Rake::Pipeline::FileWrapper} objects.
+The most important methods on a {Rake::Pipeline::FileWrapper FileWrapper} are:
+
+* {Rake::Pipeline::FileWrapper#path path}: the path of the file, relative to its input root
+* {Rake::Pipeline::FileWrapper#read read}: read the contents of the file
+* {Rake::Pipeline::FileWrapper#write write(string)}: write a String to the file
+
+For example, a simple concatenation filter would look like:
+
+  !!!ruby
+  class ConcatFilter < Rake::Pipeline::Filter
+    def generate_output(inputs, output)
+      inputs.each do |input|
+        output.write input.read
+      end
+    end
+  end
+
+If you had a series of input files like:
+
+* +app/javascripts/one.js+
+* +app/javascripts/two.js+
+* +app/javascripts/three.js+
+
+and you specified the +ConcatFilter+ in your
++AssetFile+ like:
+
+  !!!ruby
+  filter ConcatFilter, "application.js"
+
+The filter would receive a single call to
++generate_output+ with an Array of {Rake::Pipeline::FileWrapper FileWrapper}s
+representing each of the three files, and a {Rake::Pipeline::FileWrapper FileWrapper}
+representing +application.js+.
+
+== Binary Data
+
+If your filter is operating on binary data, like images,
+rather than textual data, like source code, you can specify
+that in your filter:
+
+  !!!ruby
+  class ConcatFilter < Rake::Pipeline::Filter
+    processes_binary_files
+
+    def generate_output(inputs, output)
+      inputs.each do |input|
+        output.write input.read
+      end
+    end
+  end
+
+This will stop `Rake::Pipeline` from trying to interpret the
+input files as `UTF-8`, which obviously will not work on
+binary data.
+
+= Built-In Filters
+
+At the current time, +Rake::Pipeline+ comes with a single built-in
+filter: {Rake::Pipeline::ConcatFilter}. Its implementation is
+the same as the `ConcatFilter` shown above.
+
+= Preview Server
+
+To start up the preview server, run +rakep+. This will start up
+a server that automatically recompiles files for you on the fly
+and serves up the files you need.
+
+This should allow you to have a single index.html file pointing
+at the same files in both development and production.
+
+= Compiling Assets
+
+To compile all assets before deployment, simply run:
+
+  $ rakep build
+
+= Encodings
+
+If a filter does not specify that it processes binary files,
++Rake::Pipeline+ will open all inputs and outputs as +UTF-8+.
+
+This means that if you have files encoded in other encodings,
+like +Latin-1+, +Rake::Pipeline+ will raise an exception. In
+this situation, you need to open the offending file in your
+text editor and re-save it as +UTF-8+.
+
+= Public Release Requirement
+
+Before publicly releasing this code, we need to properly support
+encodings other than UTF-8. That means using the
++default_external+ instead of hardcoding to UTF-8 and
+providing a mechanism for specifying the encoding of a file using
+a magic comment.

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+directory "doc"
+
+task :docs => Dir["lib/**"] do
+  sh "devbin/yard doc --readme README.yard --hide-void-return"
+end
+
+task :graph => ["doc", :docs] do
+  sh "devbin/yard graph --dependencies | dot -Tpng -o doc/arch.png"
+end

data/bin/rakep

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require "rake-pipeline"
+require "rake-pipeline/middleware"
+require "rack/server"
+
+module Rake
+  class Pipeline
+    class Server < Rack::Server
+      def app
+        not_found = proc { [404, { "Content-Type" => "text/plain" }, ["not found"]] }
+        config = "Assetfile"
+
+        Middleware.new(not_found, config)
+      end
+    end
+  end
+end
+
+if ARGV[0] == "build"
+  config = "Assetfile"
+  pipeline_source = File.read(config)
+  pipeline = Rake::Pipeline.class_eval "build do\n#{pipeline_source}\nend", config, 1
+  pipeline.invoke
+else
+  Rake::Pipeline::Server.new.start
+end

data/lib/rake-pipeline.rb

@@ -0,0 +1,365 @@
+require "rake-pipeline/file_wrapper"
+require "rake-pipeline/filter"
+require "rake-pipeline/filters"
+require "rake-pipeline/dsl"
+require "rake-pipeline/matcher"
+require "rake-pipeline/error"
+
+if defined?(Rails::Railtie)
+  require "rake-pipeline/railtie"
+elsif defined?(Rails)
+  require "rake-pipeline/rails_plugin"
+end
+
+require "thread"
+
+# Use the Rake namespace
+module Rake
+  # Override Rake::Task to support recursively re-enabling
+  # a task and its dependencies.
+  class Task
+
+    # @param [Rake::Application] app a Rake Application
+    # @return [void]
+    def recursively_reenable(app)
+      reenable
+
+      prerequisites.each do |dep|
+        app[dep].recursively_reenable(app)
+      end
+    end
+  end
+
+  # Override Rake::FileTask to make it sortable
+  class FileTask
+    # implement Ruby protocol for sorting
+    #
+    # @return [Fixnum]
+    def <=>(other)
+      [name, prerequisites] <=> [other.name, other.prerequisites]
+    end
+  end
+
+  # A Pipeline is responsible for taking a directory of input
+  # files, applying a number of filters to the inputs, and
+  # outputting them into an output directory.
+  #
+  # The normal way to build and configure a pipeline is by
+  # using {.build}. Inside the block passed to {.build}, all
+  # methods of {DSL} are available.
+  #
+  # @see DSL Rake::Pipeline::DSL for information on the methods
+  #   available inside the block.
+  #
+  # @example
+  #   !!!ruby
+  #   Rake::Pipeline.build do
+  #     # process all js, css and html files in app/assets
+  #     input "app/assets", "**/*.{js,coffee,css,scss,html}"
+  #
+  #     # processed files should be outputted to public
+  #     output "public"
+  #
+  #     # process all coffee files
+  #     match "*.coffee" do
+  #       # compile all CoffeeScript files. the output file
+  #       # for the compilation should be the input name
+  #       # with the .coffee extension replaced with .js
+  #       filter(CoffeeCompiler) do |input|
+  #         input.sub(/\.coffee$/, '.js')
+  #       end
+  #     end
+  #
+  #     # specify filters for js files. this includes the
+  #     # output of the previous step, which converted
+  #     # coffee files to js files
+  #     match "*.js" do
+  #       # first, wrap all JS files in a custom filter
+  #       filter ClosureFilter
+  #       # then, concatenate all JS files into a single file
+  #       filter Rake::Pipeline::ConcatFilter, "application.js"
+  #     end
+  #
+  #     # specify filters for css and scss files
+  #     match "*.{css,scss}" do
+  #       # compile CSS and SCSS files using the SCSS
+  #       # compiler. if an input file has the extension
+  #       # scss, replace it with css
+  #       filter(ScssCompiler) do |input|
+  #         input.sub(/\.scss$/, 'css')
+  #       end
+  #       # then, concatenate all CSS files into a single file
+  #       filter Rake::Pipeline::ConcatFilter, "application.css"
+  #     end
+  #
+  #     # the remaining files not specified by a matcher (the
+  #     # HTML files) are simply copied over.
+  #
+  #     # you can also specify filters here that will apply to
+  #     # all processed files (application.js and application.css)
+  #     # up until this point, as well as the HTML files.
+  #   end
+  class Pipeline
+    # @return [String] a glob representing the input files
+    attr_accessor :input_glob
+
+    # @return [String] the directory path for the input files.
+    attr_reader   :input_root
+
+    # @return [String] the directory path for the output files.
+    attr_reader   :output_root
+
+    # @return [String] the directory path for temporary files.
+    attr_reader   :tmpdir
+
+    # @return [Array] an Array of Rake::Task objects. This
+    #   property is populated by the #generate_rake_tasks
+    #   method.
+    attr_reader   :rake_tasks
+
+    # @return [String] a list of files that will be outputted
+    #   to the output directory when the pipeline is invoked
+    attr_reader   :output_files
+
+    # @return [Array] this pipeline's filters.
+    attr_reader   :filters
+
+    attr_writer :input_files
+
+    def initialize
+      @filters = []
+      @tmpdir = "tmp"
+      @mutex = Mutex.new
+    end
+
+    # Build a new pipeline taking a block. The block will
+    # be evaluated by the Rake::Pipeline::DSL class.
+    #
+    # @see Rake::Pipeline::Filter Rake::Pipeline::Filter
+    #
+    # @example
+    #   Rake::Pipeline.build do
+    #     input "app/assets"
+    #     output "public"
+    #
+    #     filter Rake::Pipeline::ConcatFilter, "app.js"
+    #   end
+    #
+    # @see DSL the Rake::Pipeline::DSL documentation.
+    #   All instance methods of DSL are available inside
+    #   the build block.
+    #
+    # @return [Rake::Pipeline] the newly configured pipeline
+    def self.build(&block)
+      pipeline = new
+      DSL.evaluate(pipeline, &block) if block
+      pipeline
+    end
+
+    @@tmp_id = 0
+
+    # Copy the current pipeline's attributes over.
+    #
+    # @param [Class] target_class the class to create a new
+    #   instance of. Defaults to the class of the current
+    #   pipeline. Is overridden in {Matcher}
+    # @param [Proc] block a block to pass to the {DSL DSL}
+    # @return [Pipeline] the new pipeline
+    # @api private
+    def copy(target_class=self.class, &block)
+      pipeline = target_class.build(&block)
+      pipeline.input_root = input_root
+      pipeline.tmpdir = tmpdir
+      pipeline.rake_application = rake_application
+      pipeline
+    end
+
+    # Set the input root of this pipeline and expand its path.
+    #
+    # @param [String] root this pipeline's input root
+    def input_root=(root)
+      @input_root = File.expand_path(root)
+    end
+
+    # Set the output root of this pipeline and expand its path.
+    #
+    # @param [String] root this pipeline's output root
+    def output_root=(root)
+      @output_root = File.expand_path(root)
+    end
+
+    # Set the temporary directory for this pipeline and expand its path.
+    #
+    # @param [String] root this pipeline's temporary directory
+    def tmpdir=(dir)
+      @tmpdir = File.expand_path(dir)
+    end
+
+    # If you specify a glob for #input_glob, this method will
+    # calculate the input files for the directory. If you supply
+    # input_files directly, this method will simply return the
+    # input_files you supplied.
+    #
+    # @return [Array<FileWrapper>] An Array of file wrappers
+    #   that represent the inputs for the current pipeline.
+    def input_files
+      return @input_files if @input_files
+
+      assert_input_provided
+
+      expanded_root = File.expand_path(input_root)
+      files = Dir[File.join(expanded_root, input_glob)].select { |f| File.file?(f) }
+
+      files.map do |file|
+        relative_path = file.sub(%r{^#{Regexp.escape(expanded_root)}/}, '')
+        FileWrapper.new(expanded_root, relative_path)
+      end
+    end
+
+    # for Pipelines, this is every file, but it may be overridden
+    # by subclasses
+    alias eligible_input_files input_files
+
+    # @return [Rake::Application] The Rake::Application to install
+    #   rake tasks onto. Defaults to Rake.application
+    def rake_application
+      @rake_application || Rake.application
+    end
+
+    # Set the rake_application on the pipeline and apply it to filters.
+    #
+    # @return [void]
+    def rake_application=(rake_application)
+      @rake_application = rake_application
+      @filters.each { |filter| filter.rake_application = rake_application }
+      @rake_tasks = nil
+    end
+
+    # Add one or more filters to the current pipeline.
+    #
+    # @param [Array<Filter>] filters a list of filters
+    # @return [void]
+    def add_filters(*filters)
+      filters.each { |filter| filter.rake_application = rake_application }
+      @filters.concat(filters)
+    end
+    alias add_filter add_filters
+
+    # Invoke the pipeline, processing the inputs into the output. If
+    # the pipeline has already been invoked, reinvoking will not
+    # pick up new input files added to the file system.
+    #
+    # @return [void]
+    def invoke
+      @mutex.synchronize do
+        self.rake_application = Rake::Application.new unless @rake_application
+
+        setup
+
+        @rake_tasks.each { |task| task.recursively_reenable(rake_application) }
+        @rake_tasks.each { |task| task.invoke }
+      end
+    end
+
+    # Pick up any new files added to the inputs and process them through
+    # the filters. Then call #invoke.
+    #
+    # @return [void]
+    def invoke_clean
+      @rake_tasks = @rake_application = nil
+      invoke
+    end
+
+    # Set up the filters and generate rake tasks. In general, this method
+    # is called by invoke.
+    #
+    # @return [void]
+    # @api private
+    def setup
+      setup_filters
+      generate_rake_tasks
+    end
+
+    # A list of the output files that invoking this pipeline will
+    # generate.
+    #
+    # @return [Array<FileWrapper>]
+    def output_files
+      @filters.last.output_files unless @filters.empty?
+    end
+
+  protected
+    # Generate a new temporary directory name.
+    #
+    # @return [String] a unique temporary directory name
+    def self.generate_tmpname
+      "rake-pipeline-tmp-#{@@tmp_id += 1}"
+    end
+
+    # Set up the filters. This will loop through all of the filters for
+    # the current pipeline and wire up their input_files and output_files.
+    #
+    # Because matchers implement the filter API, matchers will also be
+    # set up as part of this process.
+    #
+    # @return [void]
+    def setup_filters
+      last = @filters.last
+
+      @filters.inject(eligible_input_files) do |current_inputs, filter|
+        filter.input_files = current_inputs
+
+        # if filters are being reinvoked, they should keep their roots but
+        # get updated with new files.
+        filter.output_root ||= begin
+          output = if filter == last
+            output_root
+          else
+            generate_tmpdir
+          end
+
+          File.expand_path(output)
+        end
+
+        filter.setup_filters if filter.respond_to?(:setup_filters)
+
+        filter.output_files
+      end
+    end
+
+    # Generate a new temporary directory name under the main tmpdir.
+    #
+    # @return [void]
+    def generate_tmpdir
+      File.join(tmpdir, self.class.generate_tmpname)
+    end
+
+    # Generate all of the rake tasks for this pipeline.
+    #
+    # @return [void]
+    def generate_rake_tasks
+      @rake_tasks ||= begin
+        tasks = []
+
+        @filters.each do |filter|
+          # TODO: Don't generate rake tasks if we aren't
+          # creating a new Rake::Application
+          tasks = filter.generate_rake_tasks
+        end
+
+        tasks
+      end
+    end
+
+    # Assert that an input root and glob were both provided.
+    #
+    # @raise Rake::Pipeline::Error if input root or glob were missing.
+    # @return [void]
+    def assert_input_provided
+      if !input_root || !input_glob
+        raise Rake::Pipeline::Error, "You cannot get input files without " \
+                                     "first providing input files and an input root"
+      end
+    end
+  end
+end