rake-pipeline-fork 0.8.0

data/lib/rake-pipeline/manifest_entry.rb

@@ -0,0 +1,34 @@
+module Rake
+  class Pipeline
+    # Represents a single entry in a dynamic dependency {Manifest}.
+    class ManifestEntry
+      # Create a new entry from the given hash.
+      def self.from_hash(hash)
+        entry = new
+
+        entry.mtime = hash["mtime"]
+
+        hash["deps"].each do |dep, time_string|
+          entry.deps[dep] = time_string
+        end
+
+        entry
+      end
+
+      attr_accessor :deps, :mtime
+
+      def initialize(deps={}, mtime=nil)
+        @deps, @mtime = deps, mtime
+      end
+
+      def as_json
+        { :deps => @deps, :mtime => @mtime }
+      end
+
+      def ==(other)
+        mtime == other.mtime
+        deps == other.deps
+      end
+    end
+  end
+end

data/lib/rake-pipeline/matcher.rb

@@ -0,0 +1,141 @@
+require "strscan"
+
+module Rake
+  class Pipeline
+    # A Matcher is a type of pipeline that restricts its
+    # filters to a particular pattern.
+    #
+    # A Matcher's pattern is a File glob.
+    #
+    # For instance, to restrict filters to operating on
+    # JavaScript files in the +app+ directory, the Matcher's
+    # {Pipeline#inputs inputs} should include +"app"+,
+    # and its glob would be +"*.js"+.
+    #
+    # In general, you should not use Matcher directly. Instead use
+    # {DSL#match} in the block passed to {Pipeline.build}.
+    class Matcher < Pipeline
+      attr_reader :glob
+
+      # @return [Rake::Pipeline] the Rake::Pipeline that contains
+      #   this matcher.
+      attr_accessor :pipeline
+
+      # A glob matcher that a filter's input files must match
+      # in order to be processed by the filter.
+      #
+      # @return [String]
+      def glob=(pattern)
+        @glob = pattern
+        if pattern.kind_of?(Regexp)
+          @pattern = pattern
+        else
+          @pattern = scan_string
+        end
+      end
+
+      # A list of the output files that invoking this pipeline will
+      # generate. This will include the outputs of files matching
+      # the {#glob glob} and any inputs that did not match the
+      # glob.
+      #
+      # This will make those inputs available to any additional
+      # filters or matchers.
+      #
+      # @return [Array<FileWrapper>]
+      def output_files
+        super + input_files.reject do |file|
+          file.path =~ @pattern
+        end
+      end
+
+      # Override {Pipeline#finalize} to do nothing. We want to pass
+      # on our unmatched inputs to the next part of the pipeline.
+      #
+      # @return [void]
+      # @api private
+      def finalize
+      end
+
+    protected
+      # Let our containing pipeline generate temp directories for us.
+      def generate_tmpdir
+        pipeline.generate_tmpdir
+      end
+
+    private
+      # Override the default {Pipeline#eligible_input_files}
+      # to include only files that match the {#glob glob}.
+      #
+      # @return [Array<FileWrapper>]
+      def eligible_input_files
+        input_files.select do |file|
+          file.path =~ @pattern
+        end
+      end
+
+      # Convert string to regexp using StringScanner
+      #
+      # @return [Regexp]
+      def scan_string
+        scanner = StringScanner.new(glob)
+
+        output, pos = "", 0
+
+        # keep scanning until end of String
+        until scanner.eos?
+
+          # look for **/, *, {...}, or the end of the string
+          new_chars = scanner.scan_until %r{
+              \*\*/
+            | /\*\*/
+            | \*
+            | \{[^\}]*\}
+            | $
+          }x
+
+          # get the new part of the string up to the match
+          before = new_chars[0, new_chars.size - scanner.matched_size]
+
+          # get the match and new position
+          match = scanner.matched
+          pos = scanner.pos
+
+          # add any literal characters to the output
+          output << Regexp.escape(before) if before
+
+          output << case match
+          when "/**/"
+            # /**/ matches either a "/" followed by any number
+            # of characters or a single "/"
+            "(/.*|/)"
+          when "**/"
+            # **/ matches the beginning of the path or
+            # any number of characters followed by a "/"
+            "(^|.*/)"
+          when "*"
+            # * matches any number of non-"/" characters
+            "[^/]*"
+          when /\{.*\}/
+            # {...} is split over "," and glued back together
+            # as an or condition
+            "(" + match[1...-1].gsub(",", "|") + ")"
+          else String
+            # otherwise, we've grabbed until the end
+            match
+          end
+        end
+
+        if glob.include?("/")
+          # if the pattern includes a /, it must match the
+          # entire input, not just the end.
+          Regexp.new("^#{output}$", "i")
+        else
+          # anchor the pattern either at the beginning of the
+          # path or at any "/" character
+          Regexp.new("(^|/)#{output}$", "i")
+        end
+      end
+    end
+  end
+end

data/lib/rake-pipeline/middleware.rb

@@ -0,0 +1,72 @@
+require "rack"
+
+module Rake
+  class Pipeline
+    # This middleware is used to provide a server that will continuously
+    # compile your files on demand.
+    #
+    # @example
+    #   !!!ruby
+    #   use Rake::Pipeline::Middleware, Rake::Pipeline.build {
+    #     input "app/assets"
+    #     output "public"
+    #
+    #     ...
+    #   }
+    class Middleware
+      attr_accessor :project
+
+      # @param [#call] app a Rack application
+      # @param [Rake::Pipeline::Project] an existing project
+      def initialize(app, project)
+        @app = app
+        @project = project
+      end
+
+      # Automatically compiles your assets if required and
+      # serves them up.
+      #
+      # @param [Hash] env a Rack environment
+      # @return [Array(Fixnum, Hash, #each)] A rack response
+      def call(env)
+        project.invoke
+        path = env["PATH_INFO"]
+
+        if project.maps.has_key?(path)
+          return project.maps[path].call(env)
+        end
+
+        if filename = file_for(path)
+          if File.directory?(filename)
+            index = File.join(filename, "index.html")
+            filename = File.file?(index) ? index : nil
+          end
+
+          if filename
+            return response_for(filename)
+          end
+        end
+
+        @app.call(env)
+      end
+
+    private
+      def response_for(file)
+        [ 200, headers_for(file), File.open(file, "r") ]
+      end
+
+      def file_for(path)
+        project.pipelines.each do |pipeline|
+          file = Dir[File.join(pipeline.output_root, path)].sort.first
+          return file unless file.nil?
+        end
+        nil
+      end
+
+      def headers_for(path)
+        mime = Rack::Mime.mime_type(File.extname(path), "text/plain")
+        { "Content-Type" => mime }
+      end
+    end
+  end
+end

data/lib/rake-pipeline/precompile.rake

@@ -0,0 +1,8 @@
+namespace :assets do
+  desc "Precompile assets using Rake::Pipeline"
+  task :precompile do
+    config = Rails.application.config.rake_pipeline_assetfile
+    Rake::Pipeline::Project.new(config).invoke
+  end
+end
+

data/lib/rake-pipeline/project.rb

@@ -0,0 +1,335 @@
+require "digest"
+
+module Rake
+  class Pipeline
+    # A Project controls the lifecycle of a series of Pipelines,
+    # creating them from an Assetfile and recreating them if the
+    # Assetfile changes.
+    class Project
+      # @return [Pipeline] the list of pipelines in the project
+      attr_reader :pipelines
+
+      attr_reader :maps
+
+      # @return [String|nil] the path to the project's Assetfile
+      #   or nil if it was created without an Assetfile.
+      attr_reader :assetfile_path
+
+      # @return [String|nil] the digest of the Assetfile the
+      #   project was created with, or nil if the project
+      #   was created without an Assetfile.
+      attr_reader :assetfile_digest
+
+      # @return [String] the directory path for temporary files
+      attr_reader :tmpdir
+
+      # @return [String] the directory path where pipelines will
+      #   write their outputs by default
+      attr_reader :default_output_root
+
+      # @return [Array] a list of filters to be applied before
+      #   the specified filters in every pipeline
+      attr_writer :before_filters
+
+      # @return [Array] a list of filters to be applied after
+      #   the specified filters in every pipeline
+      attr_writer :after_filters
+
+      class << self
+        # Configure a new project by evaluating a block with the
+        # Rake::Pipeline::DSL::ProjectDSL class.
+        #
+        # @see Rake::Pipeline::Filter Rake::Pipeline::Filter
+        #
+        # @example
+        #   Rake::Pipeline::Project.build do
+        #     tmpdir "tmp"
+        #     output "public"
+        #
+        #     input "app/assets" do
+        #       concat "app.js"
+        #     end
+        #   end
+        #
+        # @return [Rake::Pipeline::Project] the newly configured project
+        def build(&block)
+          project = new
+          project.build(&block)
+        end
+
+        # @return [Array[String]] an array of strings that will be
+        #   appended to {#digested_tmpdir}.
+        def digest_additions
+          @digest_additions ||= []
+        end
+
+        # Set {.digest_additions} to a sorted copy of the given array.
+        def digest_additions=(additions)
+          @digest_additions = additions.sort
+        end
+
+        # Add a value to the list of strings to append to the digest
+        # temp directory. Libraries can use this to add (for example)
+        # their version numbers so that the pipeline will be rebuilt
+        # if the library version changes.
+        #
+        # @example
+        #   Rake::Pipeline::Project.add_to_digest(Rake::Pipeline::Web::Filters::VERSION)
+        #
+        # @param [#to_s] str a value to append to {#digested_tmpdir}.
+        def add_to_digest(str)
+          self.digest_additions << str.to_s
+          self.digest_additions.sort!
+        end
+      end
+
+      # @param [String|Pipeline] assetfile_or_pipeline
+      #   if this a String, create a Pipeline from the Assetfile at
+      #   that path. If it's a Pipeline, just wrap that pipeline.
+      def initialize(assetfile_or_pipeline=nil)
+        reset!
+        if assetfile_or_pipeline.kind_of?(String)
+          @assetfile_path = File.expand_path(assetfile_or_pipeline)
+          rebuild_from_assetfile(@assetfile_path)
+        elsif assetfile_or_pipeline
+          @pipelines << assetfile_or_pipeline
+        end
+      end
+
+      # Evaluate a block using the Rake::Pipeline::DSL::ProjectDSL
+      # DSL against an existing project.
+      def build(&block)
+        DSL::ProjectDSL.evaluate(self, &block) if block
+        self
+      end
+
+      # Invoke all of the project's pipelines, detecting any changes
+      # to the Assetfile and rebuilding the pipelines if necessary.
+      #
+      # @return [void]
+      # @see Rake::Pipeline#invoke
+      def invoke
+        @invoke_mutex.synchronize do
+          last_manifest.read_manifest
+
+          if dirty?
+            rebuild_from_assetfile(assetfile_path) if assetfile_dirty?
+
+            # The temporary files have to be cleaned otherwise
+            # there will be a "ghost" input. Here's an example 
+            # rake task: application.js => [a.js, b.js]. Deleting a.js
+            # will make application.js => [b.js]. The task correctly checks
+            # if B has changed (which it hasn't) and says that application.js
+            # is correct. Cleaning tmp files ensures that this doesn't happen.
+            clean if files_deleted?
+
+            pipelines.each(&:invoke)
+
+            manifest.write_manifest
+          end
+        end
+      end
+
+      # Remove the project's temporary and output files.
+      def clean
+        last_manifest.clear
+        files_to_clean.each { |file| FileUtils.rm_rf(file) }
+      end
+
+      # Clean out old tmp directories from the pipeline's
+      # {Rake::Pipeline#tmpdir}.
+      #
+      # @return [void]
+      def cleanup_tmpdir
+        obsolete_tmpdirs.each { |dir| FileUtils.rm_rf(dir) }
+      end
+
+      # Set the default output root of this project and expand its path.
+      #
+      # @param [String] root this pipeline's output root
+      def default_output_root=(root)
+        @default_output_root = File.expand_path(root)
+      end
+
+      # Set the temporary directory for this project and expand its path.
+      #
+      # @param [String] root this project's temporary directory
+      def tmpdir=(dir)
+        @tmpdir = File.expand_path(dir)
+      end
+
+      # @return [String] A subdirectory of {#tmpdir} with the digest of
+      #   the Assetfile's contents and any {.digest_additions} in its
+      #   name.
+      def digested_tmpdir
+        suffix = assetfile_digest
+        unless self.class.digest_additions.empty?
+          suffix += "-#{self.class.digest_additions.join('-')}"
+        end
+        File.join(tmpdir, "rake-pipeline-#{suffix}")
+      end
+
+      # @return Array[String] a list of the paths to temporary directories
+      #   that don't match the pipline's Assetfile digest.
+      def obsolete_tmpdirs
+        if File.directory?(tmpdir)
+          Dir["#{tmpdir}/rake-pipeline-*"].sort.reject do |dir|
+            dir == digested_tmpdir
+          end
+        else
+          []
+        end
+      end
+
+      # @return Array[String] a list of files to delete to completely clean
+      #   out a project's temporary and output files.
+      def files_to_clean
+        setup_pipelines
+        obsolete_tmpdirs + [digested_tmpdir] + output_files.map(&:fullpath)
+      end
+
+      # @return [Array[FileWrapper]] a list of the files that
+      #   will be generated when this project is invoked.
+      def output_files
+        setup_pipelines
+        pipelines.map(&:output_files).flatten
+      end
+
+      # Build a new pipeline and add it to our list of pipelines.
+      def build_pipeline(input, glob=nil, &block)
+        pipeline = Rake::Pipeline.build({
+          :before_filters => @before_filters,
+          :after_filters => @after_filters,
+          :output_root => default_output_root,
+          :tmpdir => digested_tmpdir,
+          :project => self
+        }, &block)
+
+        if input.kind_of?(Array)
+          input.each { |x| pipeline.add_input(x) }
+        elsif input.kind_of?(Hash)
+          pipeline.inputs = input
+        else
+          pipeline.add_input(input, glob)
+        end
+
+        @pipelines << pipeline
+        pipeline
+      end
+
+      # @return [Manifest] the manifest to write dependency information
+      #   to
+      def manifest
+        @manifest ||= Rake::Pipeline::Manifest.new(manifest_path)
+      end
+
+      # @return [Manifest] the manifest to write dependency information
+      #   to
+      def last_manifest
+        @last_manifest ||= Rake::Pipeline::Manifest.new(manifest_path)
+      end
+
+      # @return [String] the path to the dynamic dependency manifest
+      def manifest_path
+        File.join(digested_tmpdir, "manifest.json")
+      end
+
+    private
+      # Reset this project's internal state to the default values.
+      #
+      # @return [void]
+      def reset!
+        @pipelines = []
+        @maps = {}
+        @tmpdir = "tmp"
+        @invoke_mutex = Mutex.new
+        @default_output_root = @assetfile_digest = @assetfile_path = nil
+        @manifest = @last_manifest = nil
+      end
+
+      # Reconfigure this project based on the Assetfile at path.
+      #
+      # @param [String] path the path to the Assetfile
+      #   to use to configure the project.
+      # @param [String] source if given, this string is
+      #   evaluated instead of reading the file at assetfile_path.
+      #
+      # @return [void]
+      def rebuild_from_assetfile(path, source=nil)
+        reset!
+        source ||= File.read(path)
+        @assetfile_digest = digest(source)
+        @assetfile_path = path
+        build { instance_eval(source, path, 1) }
+      end
+
+      # Setup the pipeline so its output files will be up to date.
+      def setup_pipelines
+        pipelines.map(&:setup_filters)
+      end
+
+      # @return [String] the SHA1 digest of the given string.
+      def digest(str)
+        Digest::SHA1.hexdigest(str)
+      end
+
+      def dirty?
+        assetfile_dirty? || files_deleted? || files_dirty? 
+      end
+
+      def assetfile_dirty?
+        if assetfile_path
+          source = File.read(assetfile_path)
+          digest(source) != assetfile_digest
+        else
+          false
+        end
+      end
+
+      # Returns true if any of these conditions are met:
+      # * The pipeline hasn't been invoked yet
+      # * The input files have changed
+      # * There is a new input file
+      def files_dirty?
+        return true if manifest.empty?
+
+        previous_files = manifest.files
+
+        # check for modifications to new files
+        input_files.each do |input_file|
+          if !previous_files[input_file]
+            return true # there is a new file in the pipeline
+          elsif File.mtime(input_file).to_i != previous_files[input_file]
+            return true # existing file has been changed
+          end
+        end
+
+        false
+      end
+
+      def files_deleted?
+        last_manifest.files.each_key do |input_file|
+          return true if !File.exists?(input_file)
+        end
+
+        false
+      end
+
+      def input_files
+        static_input_files = pipelines.collect do |p|
+          p.input_files.reject { |file| file.in_directory? tmpdir }.map(&:fullpath)
+        end.flatten
+
+        dynamic_input_files = static_input_files.collect do |file|
+          if manifest[file] 
+            manifest[file].deps.keys
+          else
+            nil
+          end
+        end.flatten.compact
+
+        static_input_files + dynamic_input_files
+      end
+    end
+  end
+end