plumr 0.1

data/Rakefile

@@ -0,0 +1,43 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/clean'
+
+GEM_VERSION = "0.1"
+
+Rake::RDocTask.new do |task|
+  task.rdoc_files.add [ 'lib/**/*.rb' ]
+end
+
+task :clobber => [ :clean ]
+
+Rake::TestTask.new do |task|
+  task.ruby_opts << '-rrubygems'
+  task.libs << 'lib'
+  task.libs << 'test'
+  task.test_files = [ "test/test_all.rb" ]
+  task.verbose = true
+end
+
+gemspec = Gem::Specification.new do |gemspec|
+  gemspec.name = "plumr"
+  gemspec.version = GEM_VERSION
+  gemspec.author = "MenTaLguY <mental@rydia.net>"
+  gemspec.summary = "A builder for shell pipelines with Rake integration"
+  gemspec.test_file = 'test/test_all.rb'
+  gemspec.files = FileList[ 'Rakefile', 'test/*.rb', 'lib/**/*.rb' ]
+  gemspec.require_paths = [ 'lib' ]
+  gemspec.has_rdoc = true
+  gemspec.add_dependency 'rake'
+  gemspec.platform = Gem::Platform::RUBY
+end
+
+task :package => [ :clean, :test ]
+Rake::GemPackageTask.new( gemspec ) do |task|
+  task.gem_spec = gemspec
+  task.need_tar = true
+end
+
+task :default => [ :clean, :test ]
+

data/lib/plumr.rb

@@ -0,0 +1,295 @@
+# plumr - shell pipeline builder
+#
+# Copyright 2007  MenTaLguY <mental@rydia.net>
+#
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without 
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * The names of the authors may not be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'thread'
+require 'fileutils'
+require 'set'
+
+module Plumr
+  # OUTPUT is replaced with the filename of 
+  OUTPUT = Object.new
+
+  # Raised when command execution fails
+  class CommandError < RuntimeError
+  end
+
+  # The base interface for a Plumr "fitting", for instance Plumr::File
+  # or Plumr::Command
+  module Fitting
+    # Dumps the fitting's output to the file named by +output_file+
+    def generate(output_file)
+      raise NotImplementedError, "#{self.class}#generate not implemented"
+    end
+
+    # Disposes of any temporary files or resources that we know how to
+    # recreate later.
+    def cleanup
+      raise NotImplementedError, "#{self.class}#cleanup not implemented"
+    end
+
+    # Generates a temporary file; hints for the temporary filename are
+    # provided in the form of a +prefix+ and a numeric +index+; the actual
+    # name of created file is returned.  The file should not be deleted
+    # except via calling cleanup_temp.
+    def generate_temp(prefix, index)
+      raise NotImplementedError, "#{self.class}#generate_temp not implemented"
+    end
+
+    # Cleans up a temporary file created by generate_temp, if it is
+    # appropriate to do so; should be called on the same fixture that
+    # generated the file.
+    def cleanup_temp(filename)
+      raise NotImplementedError, "#{self.class}#cleanup_temp not implemented"
+    end
+
+    # Returns true if the fitting's output is up-to-date with respect to
+    # its inputs
+    def up_to_date?
+      raise NotImplementedError, "#{self.class}#up_to_date? not implemented"
+    end
+
+    # Returns true if the fitting's output is up-to-date as of the given
+    # timestamp
+    def older_than?(timestamp)
+      raise NotImplementedError, "#{self.class}#older_than? not implemented"
+    end
+
+    # Returns an "isolated" version of a fitting with any input Plumr::File
+    # objects replaced by their output filenames.  Returns a pair consisting
+    # of a source and a set of additional dependencies.  The source and
+    # dependencies will be either filenames or Fittings.
+    def isolate
+      raise NotImplementedError, "#{self.class}#isolate not implemented"
+    end
+  end
+
+  # A fitting which constructs a specific file
+  class File
+    include Fitting
+
+    attr_reader :filename     # the output filename
+    attr_reader :source       # the plumr fitting which generates the data
+    # any Files this file depends on, directly or indirectly
+    attr_reader :dependencies
+
+    # Creates a new Plumr::File object to generate the file +filename+;
+    # +source+ is the fixture which provides the file content; +dependencies+
+    # are any other fixtures which must be up-to-date before the file can
+    # be generated and are not part of the dependencies of +source+.
+    # Explicitly specifying dependencies is not required.
+    #
+    def initialize(filename, source=nil, *dependencies)
+      @lock = Mutex.new
+      @filename = filename.dup.freeze
+      @source = source
+      dependencies.unshift source if source
+      @dependencies = dependencies.to_set.freeze
+    end
+
+    # See Fixture#generate
+    def generate(output_file)
+      build
+      FileUtils.cp(@filename, output_file) if output_file != @filename
+      self
+    end
+
+    # Generates the file which this Plumr::File object represents, if
+    # it is not already up-to-date
+    def build
+      @dependencies.each { |d| d.build if d != @source }
+      @lock.synchronize do
+        unless up_to_date?
+          if @source
+            FileUtils.mkdir_p(::File.dirname(@filename))
+            @source.generate(@filename)
+          else
+            raise RuntimeError, "Don't know how to generate #{@filename}"
+          end
+        end
+      end
+      self
+    end
+
+    # Deletes the file and any dependencies which we know how to recreate.
+    # See Fixture#cleanup.
+    def cleanup
+      # delete only files we know how to recreate
+      return self unless @source
+      begin
+        ::File.unlink(@filename)
+        @dependencies.each { |d| d.cleanup }
+      rescue Errno::ENOENT
+      end
+      self
+    end
+
+    def generate_temp(base_name, index)
+      build
+      @filename
+    end
+
+    def cleanup_temp(filename)
+      self
+    end
+
+    def up_to_date?
+      begin
+        timestamp = ::File.mtime @filename
+      rescue Errno::ENOENT
+        return false
+      end
+      @dependencies.all? { |d| d.older_than? timestamp }
+    end
+
+    def older_than?(other_timestamp)
+      begin
+        timestamp = ::File.mtime @filename
+      rescue Errno::ENOENT
+        return false
+      end
+      timestamp < other_timestamp
+    end
+
+    def isolate
+      [ @filename, Set[self] ]
+    end
+  end
+
+  # A fitting which represents a shell command
+  class Command
+    include Fitting
+
+    attr_reader :command
+    attr_reader :args
+
+    # Creates a Command which represents the specified +command+ and
+    # a list of arguments +args+.  Arguments may be strings, fittings,
+    # or Plumr::OUTPUT.  Plumr::OUTPUT will be replaced by the name
+    # of the output file when the command is executed.
+    def initialize(command, *args) #:nodoc:
+      @command = command.dup.freeze
+      @args = args.freeze
+      @sources = @args.flatten.select { |arg| Fitting === arg }.uniq
+    end
+
+    def generate(output_file)
+      input_files = {}
+      ok = false
+      begin
+        @sources.each_with_index do |fitting, index|
+          input_files[fitting] = fitting.generate_temp(output_file, index)
+        end
+        ok = system(@command, *substitute_args(output_file, input_files, @args))
+        raise CommandError, "#{@command} exited with status #{$?}" unless ok
+      ensure
+        cleanup_temp(output_file) unless ok
+        input_files.each_pair do |fitting, input_file|
+          fitting.cleanup_temp(input_file)
+        end
+      end
+      self
+    end
+
+    def cleanup
+      @sources.each { |source| source.cleanup }
+      self
+    end
+
+    def generate_temp(prefix, index)
+      # FIXME: need better tempfile generation, but I'm loath to use stdlib's
+      # Thread.critical-based tempfile stuff
+      output_file = "#{prefix.sub(/\.plumr-temp$/,'')}.#{index}.plumr-temp"
+      generate(output_file)
+      output_file
+    end
+
+    def cleanup_temp(filename)
+      begin
+        ::File.unlink(filename)
+      rescue Exception
+      end
+      self
+    end
+
+    def up_to_date?
+      @sources.all? { |source| source.up_to_date? }
+    end
+
+    def older_than?(timestamp)
+      @sources.all? { |source| source.older_than? timestamp }
+    end
+
+    def isolate
+      dependencies = Set.new
+      args = []
+      @args.each do |arg|
+        case arg
+        when Fitting
+          arg_source, arg_dependencies = arg.isolate
+          dependencies.merge arg_dependencies
+          args.push arg_source
+        else
+          args.push arg
+        end
+      end
+      [ with_replaced_args(*args), dependencies ]
+    end
+
+private
+    def with_replaced_args(*args) #:nodoc:
+      self.class.new(@command, *args)
+    end
+
+    def substitute_args(output_file, input_files, args) #:nodoc:
+      args.map { |arg|
+        case arg
+        when OUTPUT
+          output_file
+        when Fitting
+          input_files[arg]
+        when Array
+          substitute_args(output_file, input_files, arg).to_s
+        else
+          arg
+        end
+      }
+    end
+  end
+
+  # Creates a new Plumr::File
+  def self.file(*args)
+    Plumr::File.new(*args)
+  end
+
+  # Creates a new Plumr::Command
+  def self.command(*args)
+    Plumr::Command.new(*args)
+  end
+end
+

data/lib/plumr/rake.rb

@@ -0,0 +1,65 @@
+# plumr/rake - Rake integration for Plumr
+#
+# Copyright 2007-2008  MenTaLguY <mental@rydia.net>
+#
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without 
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * The names of the authors may not be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'plumr'
+require 'set'
+require 'rake'
+
+module Plumr
+
+class File
+  @rake_tasks = Set.new
+  def self.rake_tasks ; @rake_tasks ; end
+
+  def rakeify
+    rake_tasks = File.rake_tasks
+    return self if rake_tasks.include? @filename
+
+    if @source
+      rake_tasks.add @filename
+      source, dependencies = @source.isolate
+      dependencies.each { |d| d.rakeify }
+      dep_files = dependencies.map { |d| d.filename }
+      case source
+      when String
+        Rake::FileTask.define_task(@filename, *dep_files) do
+          cp source, @filename
+        end
+      else
+        Rake::FileTask.define_task(@filename, *dep_files) do
+          source.generate @filename
+        end
+      end
+    end
+
+    self
+  end
+end
+
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: plumr
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+date: 2008-02-06 00:00:00 -05:00
+summary: A builder for shell pipelines with Rake integration
+require_paths: 
+- lib
+email: 
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+- MenTaLguY <mental@rydia.net>
+files: 
+- Rakefile
+- test/test_all.rb
+- lib/plumr.rb
+- lib/plumr/rake.rb
+test_files: 
+- test/test_all.rb
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: