rake-pipeline 0.5.0 → 0.7.0

data/lib/rake-pipeline/manifest.rb

@@ -0,0 +1,63 @@
+require 'json'
+
+module Rake
+  class Pipeline
+    # A Manifest is a container for storing dynamic dependency information.
+    # A {DynamicFileTask} will use a {Manifest} to keep track of its dynamic
+    # dependencies. This allows us to avoid scanning a file for dynamic
+    # dependencies if its contents have not changed.
+    class Manifest
+      attr_accessor :entries
+      attr_accessor :manifest_file
+
+      def initialize(manifest_file="manifest.json")
+        @manifest_file ||= manifest_file
+        @entries = {}
+      end
+
+      # Get the manifest off the file system, if it exists.
+      def read_manifest
+        @entries = File.file?(manifest_file) ? JSON.parse(File.read(manifest_file)) : {}
+
+        # convert the manifest JSON into a Hash of ManifestEntry objects
+        @entries.each do |file, raw|
+          @entries[file] = Rake::Pipeline::ManifestEntry.from_hash(raw)
+        end
+
+        self
+      end
+
+      # Write a JSON representation of this manifest out to disk if we
+      # have entries to save.
+      def write_manifest
+        unless @entries.empty?
+          File.open(manifest_file, "w") do |file|
+            file.puts JSON.generate(as_json)
+          end
+        end
+      end
+
+      # Convert this Manifest into a hash suitable for converting to
+      # JSON.
+      def as_json
+        hash = {}
+
+        @entries.each do |name, entry|
+          hash[name] = entry.as_json
+        end
+
+        hash
+      end
+
+      # Look up an entry by filename.
+      def [](key)
+        @entries[key]
+      end
+
+      # Set an entry
+      def []=(key, value)
+        @entries[key] = value
+      end
+    end
+  end
+end

data/lib/rake-pipeline/manifest_entry.rb

@@ -0,0 +1,34 @@
+module Rake
+  class Pipeline
+    # Represents a single entry in a dynamic dependency {Manifest}.
+    class ManifestEntry
+      # Create a new entry from the given hash.
+      def self.from_hash(hash)
+        entry = new
+
+        entry.mtime = DateTime.parse(hash["mtime"]).to_time
+
+        hash["deps"].each do |dep, time_string|
+          entry.deps[dep] = DateTime.parse(time_string).to_time
+        end
+
+        entry
+      end
+
+      attr_accessor :deps, :mtime
+
+      def initialize(deps={}, mtime=nil)
+        @deps, @mtime = deps, mtime
+      end
+
+      def as_json
+        { :deps => @deps, :mtime => @mtime }
+      end
+
+      def ==(other)
+        mtime == other.mtime
+        deps == other.deps
+      end
+    end
+  end
+end

data/lib/rake-pipeline/matcher.rb

@@ -9,7 +9,7 @@ module Rake
     #
     # For instance, to restrict filters to operating on
     # JavaScript files in the +app+ directory, the Matcher's
-    # {Pipeline#input_root input_root} should be +"app"+,
+    # {Pipeline#inputs inputs} should include +"app"+,
     # and its glob would be +"*.js"+.
     #
     # In general, you should not use Matcher directly. Instead use
@@ -17,13 +17,68 @@ module Rake
     class Matcher < Pipeline
       attr_reader :glob
 
+      # @return [Rake::Pipeline] the Rake::Pipeline that contains
+      #   this matcher.
+      attr_accessor :pipeline
+
       # A glob matcher that a filter's input files must match
       # in order to be processed by the filter.
       #
       # @return [String]
       def glob=(pattern)
         @glob = pattern
-        scanner = StringScanner.new(pattern)
+        if pattern.kind_of?(Regexp)
+          @pattern = pattern
+        else
+          @pattern = scan_string
+        end
+      end
+
+      # A list of the output files that invoking this pipeline will
+      # generate. This will include the outputs of files matching
+      # the {#glob glob} and any inputs that did not match the
+      # glob.
+      #
+      # This will make those inputs available to any additional
+      # filters or matchers.
+      #
+      # @return [Array<FileWrapper>]
+      def output_files
+        super + input_files.reject do |file|
+          file.path =~ @pattern
+        end
+      end
+
+      # Override {Pipeline#finalize} to do nothing. We want to pass
+      # on our unmatched inputs to the next part of the pipeline.
+      #
+      # @return [void]
+      # @api private
+      def finalize
+      end
+
+    protected
+      # Let our containing pipeline generate temp directories for us.
+      def generate_tmpdir
+        pipeline.generate_tmpdir
+      end
+
+    private
+      # Override the default {Pipeline#eligible_input_files}
+      # to include only files that match the {#glob glob}.
+      #
+      # @return [Array<FileWrapper>]
+      def eligible_input_files
+        input_files.select do |file|
+          file.path =~ @pattern
+        end
+      end
+
+      # Convert string to regexp using StringScanner
+      #
+      # @return [Regexp]
+      def scan_string
+        scanner = StringScanner.new(glob)
 
         output, pos = "", 0
 
@@ -71,34 +126,14 @@ module Rake
           end
         end
 
-        # anchor the pattern either at the beginning of the
-        # path or at any "/" character
-        @pattern = Regexp.new("(^|/)#{output}$", "i")
-      end
-
-      # A list of the output files that invoking this pipeline will
-      # generate. This will include the outputs of files matching
-      # the {#glob glob} and any inputs that did not match the
-      # glob.
-      #
-      # This will make those inputs available to any additional
-      # filters or matchers.
-      #
-      # @return [Array<FileWrapper>]
-      def output_files
-        super + input_files.reject do |file|
-          file.path =~ @pattern
-        end
-      end
-
-    private
-      # Override the default {Pipeline#eligible_input_files}
-      # to include only files that match the {#glob glob}.
-      #
-      # @return [Array<FileWrapper>]
-      def eligible_input_files
-        input_files.select do |file|
-          file.path =~ @pattern
+        if glob.include?("/")
+          # if the pattern includes a /, it must match the
+          # entire input, not just the end.
+          Regexp.new("^#{output}$", "i")
+        else
+          # anchor the pattern either at the beginning of the
+          # path or at any "/" character
+          Regexp.new("(^|/)#{output}$", "i")
         end
       end
     end

data/lib/rake-pipeline/middleware.rb

@@ -14,19 +14,14 @@ module Rake
     #     ...
     #   }
     class Middleware
-      attr_accessor :pipeline
+      attr_accessor :project
 
-      # @param [#call] a Rack application
-      # @param [Pipeline] a Rake::Pipeline
+      # @param [#call] app a Rack application
+      # @param [String|Rake::Pipeline] pipeline either a path to an
+      #   Assetfile to use to build a pipeline, or an existing pipeline.
       def initialize(app, pipeline)
         @app = app
-
-        if pipeline.is_a?(String)
-          pipeline_source = File.read(pipeline)
-          pipeline = Pipeline.class_eval "build do\n#{pipeline_source}\nend", pipeline, 1
-        end
-
-        @pipeline = pipeline
+        @project = Rake::Pipeline::Project.new(pipeline)
       end
 
       # Automatically compiles your assets if required and
@@ -35,9 +30,13 @@ module Rake
       # @param [Hash] env a Rack environment
       # @return [Array(Fixnum, Hash, #each)] A rack response
       def call(env)
-        pipeline.invoke_clean
+        project.invoke_clean
         path = env["PATH_INFO"]
 
+        if project.maps.has_key?(path)
+          return project.maps[path].call(env)
+        end
+
         if filename = file_for(path)
           if File.directory?(filename)
             index = File.join(filename, "index.html")
@@ -58,7 +57,11 @@ module Rake
       end
 
       def file_for(path)
-        Dir[File.join(pipeline.output_root, path)].first
+        project.pipelines.each do |pipeline|
+          file = Dir[File.join(pipeline.output_root, path)].sort.first
+          return file unless file.nil?
+        end
+        nil
       end
 
       def headers_for(path)

data/lib/rake-pipeline/precompile.rake

@@ -0,0 +1,8 @@
+namespace :assets do
+  desc "Precompile assets using Rake::Pipeline"
+  task :precompile do
+    config = Rails.application.config.rake_pipeline_assetfile
+    Rake::Pipeline::Project.new(config).invoke
+  end
+end
+

data/lib/rake-pipeline/project.rb

@@ -0,0 +1,280 @@
+require "digest"
+
+module Rake
+  class Pipeline
+    # A Project controls the lifecycle of a series of Pipelines,
+    # creating them from an Assetfile and recreating them if the
+    # Assetfile changes.
+    class Project
+      # @return [Pipeline] the list of pipelines in the project
+      attr_reader :pipelines
+
+      attr_reader :maps
+
+      # @return [String|nil] the path to the project's Assetfile
+      #   or nil if it was created without an Assetfile.
+      attr_reader :assetfile_path
+
+      # @return [String|nil] the digest of the Assetfile the
+      #   project was created with, or nil if the project
+      #   was created without an Assetfile.
+      attr_reader :assetfile_digest
+
+      # @return [String] the directory path for temporary files
+      attr_reader :tmpdir
+
+      # @return [String] the directory path where pipelines will
+      #   write their outputs by default
+      attr_reader :default_output_root
+
+      # @return [Array] a list of filters to be applied before
+      #   the specified filters in every pipeline
+      attr_writer :before_filters
+
+      # @return [Array] a list of filters to be applied after
+      #   the specified filters in every pipeline
+      attr_writer :after_filters
+
+      class << self
+        # Configure a new project by evaluating a block with the
+        # Rake::Pipeline::DSL::ProjectDSL class.
+        #
+        # @see Rake::Pipeline::Filter Rake::Pipeline::Filter
+        #
+        # @example
+        #   Rake::Pipeline::Project.build do
+        #     tmpdir "tmp"
+        #     output "public"
+        #
+        #     input "app/assets" do
+        #       concat "app.js"
+        #     end
+        #   end
+        #
+        # @return [Rake::Pipeline::Project] the newly configured project
+        def build(&block)
+          project = new
+          project.build(&block)
+        end
+
+        # @return [Array[String]] an array of strings that will be
+        #   appended to {#digested_tmpdir}.
+        def digest_additions
+          @digest_additions ||= []
+        end
+
+        # Set {.digest_additions} to a sorted copy of the given array.
+        def digest_additions=(additions)
+          @digest_additions = additions.sort
+        end
+
+        # Add a value to the list of strings to append to the digest
+        # temp directory. Libraries can use this to add (for example)
+        # their version numbers so that the pipeline will be rebuilt
+        # if the library version changes.
+        #
+        # @example
+        #   Rake::Pipeline::Project.add_to_digest(Rake::Pipeline::Web::Filters::VERSION)
+        #
+        # @param [#to_s] str a value to append to {#digested_tmpdir}.
+        def add_to_digest(str)
+          self.digest_additions << str.to_s
+          self.digest_additions.sort!
+        end
+      end
+
+      # @param [String|Pipeline] assetfile_or_pipeline
+      #   if this a String, create a Pipeline from the Assetfile at
+      #   that path. If it's a Pipeline, just wrap that pipeline.
+      def initialize(assetfile_or_pipeline=nil)
+        reset!
+        if assetfile_or_pipeline.kind_of?(String)
+          @assetfile_path = File.expand_path(assetfile_or_pipeline)
+          rebuild_from_assetfile(@assetfile_path)
+        elsif assetfile_or_pipeline
+          @pipelines << assetfile_or_pipeline
+        end
+      end
+
+      # Evaluate a block using the Rake::Pipeline::DSL::ProjectDSL
+      # DSL against an existing project.
+      def build(&block)
+        DSL::ProjectDSL.evaluate(self, &block) if block
+        self
+      end
+
+      # Invoke all of the project's pipelines.
+      #
+      # @see Rake::Pipeline#invoke
+      def invoke
+        @invoke_mutex.synchronize do
+          pipelines.each(&:invoke)
+          manifest.write_manifest
+        end
+      end
+
+      # Invoke all of the project's pipelines, detecting any changes
+      # to the Assetfile and rebuilding the pipelines if necessary.
+      #
+      # @return [void]
+      # @see Rake::Pipeline#invoke_clean
+      def invoke_clean
+        @invoke_mutex.synchronize do
+          if assetfile_path
+            source = File.read(assetfile_path)
+            if digest(source) != assetfile_digest
+              rebuild_from_assetfile(assetfile_path, source)
+            end
+          end
+          pipelines.each(&:invoke_clean)
+          manifest.write_manifest
+        end
+      end
+
+      # Remove the project's temporary and output files.
+      def clean
+        files_to_clean.each { |file| FileUtils.rm_rf(file) }
+      end
+
+      # Clean out old tmp directories from the pipeline's
+      # {Rake::Pipeline#tmpdir}.
+      #
+      # @return [void]
+      def cleanup_tmpdir
+        obsolete_tmpdirs.each { |dir| FileUtils.rm_rf(dir) }
+      end
+
+      # Set the default output root of this project and expand its path.
+      #
+      # @param [String] root this pipeline's output root
+      def default_output_root=(root)
+        @default_output_root = File.expand_path(root)
+      end
+
+      # Set the temporary directory for this project and expand its path.
+      #
+      # @param [String] root this project's temporary directory
+      def tmpdir=(dir)
+        @tmpdir = File.expand_path(dir)
+      end
+
+      # @return [String] A subdirectory of {#tmpdir} with the digest of
+      #   the Assetfile's contents and any {.digest_additions} in its
+      #   name.
+      def digested_tmpdir
+        suffix = assetfile_digest
+        unless self.class.digest_additions.empty?
+          suffix += "-#{self.class.digest_additions.join('-')}"
+        end
+        File.join(tmpdir, "rake-pipeline-#{suffix}")
+      end
+
+      # @return Array[String] a list of the paths to temporary directories
+      #   that don't match the pipline's Assetfile digest.
+      def obsolete_tmpdirs
+        if File.directory?(tmpdir)
+          Dir["#{tmpdir}/rake-pipeline-*"].sort.reject do |dir|
+            dir == digested_tmpdir
+          end
+        else
+          []
+        end
+      end
+
+      # @return Array[String] a list of files to delete to completely clean
+      #   out a project's temporary and output files.
+      def files_to_clean
+        setup_pipelines
+        obsolete_tmpdirs + [digested_tmpdir] + output_files.map(&:fullpath)
+      end
+
+      # @return [Array[FileWrapper]] a list of the files that
+      #   will be generated when this project is invoked.
+      def output_files
+        setup_pipelines
+        pipelines.map(&:output_files).flatten
+      end
+
+      # Build a new pipeline and add it to our list of pipelines.
+      def build_pipeline(input, glob=nil, &block)
+        pipeline = Rake::Pipeline.build({
+          :before_filters => @before_filters,
+          :after_filters => @after_filters,
+          :output_root => default_output_root,
+          :tmpdir => digested_tmpdir,
+          :project => self
+        }, &block)
+
+        if input.kind_of?(Array)
+          input.each { |x| pipeline.add_input(x) }
+        elsif input.kind_of?(Hash)
+          pipeline.inputs = input
+        else
+          pipeline.add_input(input, glob)
+        end
+
+        @pipelines << pipeline
+        pipeline
+      end
+
+      # @return [Manifest] the manifest to read dependency information
+      #   from
+      def last_manifest
+        @last_manifest ||= begin
+          m = Rake::Pipeline::Manifest.new(manifest_path)
+          m.read_manifest
+        end
+      end
+
+      # @return [Manifest] the manifest to write dependency information
+      #   to
+      def manifest
+        @manifest ||= Rake::Pipeline::Manifest.new(manifest_path)
+      end
+
+      # @return [String] the path to the dynamic dependency manifest
+      def manifest_path
+        File.join(digested_tmpdir, "manifest.json")
+      end
+
+    private
+      # Reset this project's internal state to the default values.
+      #
+      # @return [void]
+      def reset!
+        @pipelines = []
+        @maps = {}
+        @tmpdir = "tmp"
+        @invoke_mutex = Mutex.new
+        @default_output_root = @assetfile_digest = @assetfile_path = nil
+        @manifest = @last_manifest = nil
+      end
+
+      # Reconfigure this project based on the Assetfile at path.
+      #
+      # @param [String] path the path to the Assetfile
+      #   to use to configure the project.
+      # @param [String] source if given, this string is
+      #   evaluated instead of reading the file at assetfile_path.
+      #
+      # @return [void]
+      def rebuild_from_assetfile(path, source=nil)
+        reset!
+        source ||= File.read(path)
+        @assetfile_digest = digest(source)
+        @assetfile_path = path
+        build { instance_eval(source, path, 1) }
+      end
+
+      # Setup the pipeline so its output files will be up to date.
+      def setup_pipelines
+        pipelines.map(&:setup_filters)
+      end
+
+      # @return [String] the SHA1 digest of the given string.
+      def digest(str)
+        Digest::SHA1.hexdigest(str)
+      end
+    end
+  end
+end