ripe 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13032eb238736b94168961b94cfd79896ad89ffd
-  data.tar.gz: 6951a4e3da93749f18e44618f7ddc617edafad44
+  metadata.gz: c591b62a4a100240062bdebd04a0a8f3bec68f6a
+  data.tar.gz: 7c976775445a90aac08c9b24d4bcfa660be4ac7f
 SHA512:
-  metadata.gz: b3538313e05b7189027942f553b8886993ff361b49b99b61c8ca91af91baf319e95f51d511bb32556501ca3b7caa5d499d7acda98bf4e7318265596c2ece29c8
-  data.tar.gz: 733a2265845c4c8049756d9e06f23337a034714a97d19d232db3c7052610eb322d23f5b79a55273ba95f9f40e95ba9fd5217ce50f9b4cd2552c498d1fa73696d
+  metadata.gz: 12e2e041891e65c853a7fbeb2b992bb8c47c7c6d947d49e686fcf62984ff5b2ca70b51d9a7c0c1c1fc9b612a8abd859a0d8bfae9c019e3d5bcf3addb81204324
+  data.tar.gz: 2a2747b8c3f31a32e329f368422664c7c2d7e0e1224e8d4bf6664e4865d9c8b07c23ef14a66d77170cc503fcd42de38b08d0154a461da821cb6b044b82845d57

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.5
+2.4.0

data/.travis.yml

@@ -1,4 +1,7 @@
 language: ruby
+rvm:
+  - ruby-2.4.0
+  - ruby-head
 cache: bundler
 script: bundle exec rake spec
 notifications:

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+HEAD
+----
+
+v0.3.0
+------
+
+  - [ndejay/ripe#13](http://github.com/ndejay/ripe/issues/13): Change the permissions of the database to the same as the user's umask.
+  - [ndejay/ripe#10](http://github.com/ndejay/ripe/issues/10): Support config file for workflows.
+  - [ndejay/ripe#8](http://github.com/ndejay/ripe/issues/8): Support ERB task templates.
+  - [ndejay/ripe#6](http://github.com/ndejay/ripe/issues/6): Track project_name and user in the ripe metadata db.
+  - Track status for workers started locally.
+  - Automatically create accessors for `DB::Worker#status`.
+
 v0.2.2
 ------
 

data/lib/ripe/blocks.rb

@@ -10,4 +10,6 @@ require_relative 'blocks/block'
 require_relative 'blocks/parallel_block'
 require_relative 'blocks/serial_block'
 require_relative 'blocks/working_block'
+require_relative 'blocks/bash_block'
+require_relative 'blocks/erb_block'
 require_relative 'blocks/liquid_block'

data/lib/ripe/blocks/bash_block.rb

@@ -0,0 +1,107 @@
+require 'liquid'
+
+module Ripe
+
+  module Blocks
+
+    ##
+    # This class represents a working block that should be processed using the
+    # Bash adaptor block.  In +ripe+ <= 0.2.1, this templating system was the
+    # default behaviour of {WorkingBlock}.
+    #
+    # Keys with string values in the format of:
+    #
+    #     vars["some_key"] = "value"
+    #
+    # are converted as follows:
+    #
+    #     SOME_KEY="value"
+    #
+    # Keys with array values in the format of:
+    #
+    #     vars["some_key"] = ["one", "two"]
+    #
+    # are converted as follows:
+    #
+    #     SOME_KEY=("one" "two")
+    #
+    # @see Ripe::Blocks::WorkingBlock
+
+    class BashBlock < WorkingBlock
+
+      ##
+      # Create a new, empty {BashBlock}.
+      #
+      # @param filename [String] filename of the template file
+      # @param vars [Hash<Symbol, String>] key-value pairs
+
+      def initialize(filename, vars = {})
+        super(filename, vars)
+      end
+
+      ##
+      # Return working block +parameters+ as a sequence of bash variable
+      # assignments.
+      #
+      # @return [String] sequence of bash variable assignments
+
+      def declarations
+        vars.map do |key, value|
+          lh = key.upcase
+          rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
+          "#{lh}=#{rh}"
+        end
+      end
+
+      ##
+      # (see Block#command)
+      #
+      # The resulting string contains the result of the application of
+      # parameters to the +task+ from which the {BashBlock} was defined.
+      #
+      # @see Ripe::DB::Task
+      # @see Ripe::DSL::TaskDSL
+
+      def command
+        <<-EOF.gsub(/^[ ]+/, '')
+
+        # <#{id}>
+
+        #{declarations.join("\n")}
+
+        exec 1>"$LOG" 2>&1
+
+        #{File.new(@filename).read}
+        echo "##.DONE.##"
+
+        # </#{id}>
+        EOF
+      end
+
+      ##
+      # Return string handle for referring to this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+      
+      def self.id
+        'bash'
+      end
+
+      ##
+      # Return expected file extension type for this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+
+      def self.extension
+        'sh'
+      end
+
+    end
+
+  end
+
+end

data/lib/ripe/blocks/erb_block.rb

@@ -0,0 +1,89 @@
+require 'erb'
+
+module Ripe
+
+  module Blocks
+
+    ##
+    # This class represents a working block that should be processed using the
+    # ERB templating system.
+    #
+    # Keys defined as:
+    #
+    #     vars["some_key"] = "value"
+    #
+    # can be substituted in the ERB template using:
+    #
+    #     <%= vars.some_key %>
+    #
+    # @see Ripe::Blocks::ERBBlock
+
+    class ERBBlock < WorkingBlock
+      #
+      ##
+      # Create a new, empty {LiquidBlock}.
+      #
+      # @param filename [String] filename of the template file
+      # @param vars [Hash<Symbol, String>] key-value pairs
+
+      def initialize(filename, vars = {})
+        super(filename, vars)
+      end
+
+      ##
+      # (see Block#command)
+      #
+      # The resulting string contains the render result of the liquid template
+      # based on the parameters specified in +vars+.
+
+      def command
+        vars = @vars
+        vars.define_singleton_method(:get_binding) { binding } # Expose private method
+        vars.define_singleton_method(:method_missing) do |name|
+          return self[name] if key? name
+          self.each { |k,v| return v if k.to_s.to_sym == name }
+          # super.method_missing name
+        end
+
+        template = <<-EOF.gsub(/^[ ]+/, '')
+
+        # <#{id}>
+
+        exec 1>"<%= vars.log %>" 2>&1
+
+        #{File.new(@filename).read}
+        echo "##.DONE.##"
+
+        # </#{id}>
+        EOF
+
+        ERB.new(template).result(vars.get_binding)
+      end
+
+      ##
+      # Return string handle for referring to this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+
+      def self.id
+        'erb'
+      end
+
+      ##
+      # Return expected file extension type for this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+
+      def self.extension
+        'sh.erb'
+      end
+
+    end
+
+  end
+
+end

data/lib/ripe/blocks/liquid_block.rb

@@ -6,8 +6,7 @@ module Ripe
 
     ##
     # This class represents a working block that should be processed using the
-    # Liquid templating engine, rather than the simple +bash+ engine defined in
-    # {WorkingBlock}.
+    # Liquid templating engine.
     #
     # @see Ripe::Blocks::WorkingBlock
 
@@ -43,6 +42,28 @@ module Ripe
         template.render(declarations)
       end
 
+      ##
+      # Return string handle for referring to this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+      
+      def self.id
+        'liquid'
+      end
+
+      ##
+      # Return expected file extension type for this type of `WorkingBlock`.
+      #
+      # @see Ripe::DSL::TaskDSL
+      #
+      # @return [String]
+
+      def self.extension
+        'sh.liquid'
+      end
+
     end
 
   end

data/lib/ripe/blocks/working_block.rb

@@ -4,10 +4,15 @@ module Ripe
 
     ##
     # This class represents a {Ripe::CLI::TaskCLI} that has been parametrized.
-    # In the block arborescence, {WorkingBlock}s are always leaf nodes.
+    # In the block arborescence, {WorkingBlock}s are always leaf nodes and therefore
+    # contain scripts to be executed.
+    #
+    # In order version of ripe (<= 0.2.1), {WorkingBlock} and {BashBlock} were
+    # considered the same entity.
     #
     # @see Ripe::CLI::TaskCLI
     # @see Ripe::WorkerController::Preparer#prepare
+    # @see Ripe::Blocks::BashBlock
 
     class WorkingBlock < Block
 
@@ -22,45 +27,6 @@ module Ripe
         super(File.basename(@filename), [], vars)
       end
 
-      ##
-      # Return working block +parameters+ as a sequence of bash variable
-      # assignments.
-      #
-      # @return [String] sequence of bash variable assignments
-
-      def declarations
-        vars.map do |key, value|
-          lh = key.upcase
-          rh = value.is_a?(Array) ? "(\"#{value.join("\" \"")}\")" : "\"#{value}\""
-          "#{lh}=#{rh}"
-        end
-      end
-
-      ##
-      # (see Block#command)
-      #
-      # The resulting string contains the result of the application of
-      # parameters to the +task+ from which the {WorkingBlock} was defined.
-      #
-      # @see Ripe::DB::Task
-      # @see Ripe::DSL::TaskDSL
-
-      def command
-        <<-EOF.gsub(/^[ ]+/, '')
-
-        # <#{id}>
-
-        #{declarations.join("\n")}
-
-        exec 1>"$LOG" 2>&1
-
-        #{File.new(@filename).read}
-        echo "##.DONE.##"
-
-        # </#{id}>
-        EOF
-      end
-
       ##
       # (see Block#prune)
       #
@@ -96,6 +62,15 @@ module Ripe
         [@id]
       end
 
+      ##
+      # Collect a list of classes derived from `WorkingBlock`.
+      #
+      # @return Array<Class>
+
+      def self.subclasses
+        ObjectSpace.each_object(Class).select { |klass| klass < self }
+      end
+
     end
 
   end

data/lib/ripe/cli.rb

@@ -71,6 +71,8 @@ module Ripe
     desc 'prepare SAMPLES', 'Prepare jobs from template workflow'
     option :workflow, :aliases => '-w', :type => :string, :required => true,
       :desc => 'Workflow to be applied'
+    option :config, :aliases => '-c', :type => :string, :required => false,
+      :desc => 'Config file for workflows.'
     option :options, :aliases => '-o', :type => :string, :required => false,
       :desc => 'Options', :default => ''
     option :start, :aliases => '-s', :type => :boolean, :required => false,
@@ -92,8 +94,11 @@ module Ripe
 
       abort 'No samples specified.' if samples.length == 0
 
-      workers = repo.controller.prepare(options[:workflow], samples,
-                                        Helper.parse_cli_opts(options[:options]))
+      config = options[:config] ? Helper.parse_config(options[:config]) : {}
+      workflow_options = config[options[:workflow].to_sym] ||= {}
+      workflow_options.merge!(Helper.parse_cli_opts(options[:options]))
+
+      workers = repo.controller.prepare(options[:workflow], samples, workflow_options)
 
       repo.controller.start(workers) if options[:start]
     end

data/lib/ripe/cli/helper.rb

@@ -24,6 +24,27 @@ module Ripe
         params.inject(&:merge) || {}
       end
 
+      ##
+      # Read and parse a json configuration file into a symbolized hash with
+      # the content of that file.
+      #
+      # @param filename [String] name of a configuration file in json format
+      # @return [Hash] a symbolized hash of the content of the json file, or
+      # nothing if the file was not found or ill defined.
+
+      def self.parse_config(filename)
+        begin
+          file = File.read(filename)
+          begin
+            JSON.parse(file, :symbolize_names => true)
+          rescue
+            abort "Configuration file found but ill defined: #{filename}"
+          end
+        rescue
+          abort "Configuration file specified but not found: #{filename}"
+        end
+      end
+
     end
 
   end

data/lib/ripe/db/worker.rb

@@ -57,6 +57,18 @@ module Ripe
         FileUtils.rm_r dir if Dir.exists? dir
       end
 
+      # Automatically create accessors for `#status`.
+      #
+      #     worker.status == :prepared
+      #
+      # becomes
+      #
+      #     worker.prepared?
+
+      [:unprepared, :prepared, :queueing, :idle,
+       :blocked, :active, :active_local, :cancelled, :completed].map do |s|
+        define_method("#{s}?") { status.to_s == s.to_s }
+      end
     end
 
   end

data/lib/ripe/db/worker_migration.rb

@@ -24,6 +24,8 @@ module Ripe
           t.string  :time
           t.string  :status, default: :unprepared
           t.string  :walltime
+          t.string  :user
+          t.string  :project_name
         end
       end
 

data/lib/ripe/dsl/task_dsl.rb

@@ -22,6 +22,10 @@ module Ripe
     #      t.param :param2, 'val2'
     #    end
     #
+    #    foo = task 'foo', task: 'bash' do |t|
+    #      t.param :param1, 'val1'
+    #    end
+    #
     # It internally uses +Ripe::DSL::TaskDSL+ to provide the DSL.
     #
     # @see Ripe::DSL::TaskDSL
@@ -32,12 +36,21 @@ module Ripe
     # @param block [Proc] executes block in the context of +TaskDSL+
     # @return [WorkingBlock, nil]
 
-    def task(handle, &block)
-      filename = Library.find(:task, handle)
-      abort "Could not find task #{handle}." if filename == nil
+    def task(handle, vars = {type: 'bash'}, &block)
+      # Obtain list of `WorkingBlock` types.
+      working_block = Blocks::WorkingBlock.subclasses.select { |k| k.id == vars[:type] }.first
+
+      filename = "#{handle}.#{working_block.extension}"
+      full_filename = Library.find(:task, filename)
+
+      if full_filename == nil
+        abort "Could not find task `#{filename}`."
+      # else
+      #   puts "Found task `#{handle}` @ `#{full_filename}`."
+      end
 
       params = TaskDSL.new(handle, &block).params
-      Blocks::WorkingBlock.new(filename, params)
+      working_block.new(full_filename, params)
     end
 
     ##

data/lib/ripe/library.rb

@@ -27,17 +27,14 @@ module Ripe
       #
       # @param comp [Symbol] Type of component: either +:workflow+ or
       #   +:task+.
-      # @param handle [String] Name of component
+      # @param filename [String] Filename of component
       # @return [String, nil] Full path of the component if found, and +nil+
       #   otherwise.
  
-      def find(comp, handle)
-        ext = { task:     'sh',
-                workflow: 'rb' }
-
+      def find(comp, filename)
         search = paths.map do |path|
-          filename = "#{path}/#{comp}s/#{handle}.#{ext[comp]}"
-          (File.exists? filename) ? filename : nil
+          file = "#{path}/#{comp}s/#{filename}"
+          (File.exists? file) ? file : nil
         end
 
         search.compact.first

data/lib/ripe/repo.rb

@@ -74,6 +74,9 @@ module Ripe
         # Create the tables
         DB::WorkerMigration.up
         DB::TaskMigration.up
+
+        # Set the database's permissions to the user's umask
+        FileUtils.chmod(0666 - File.umask(), DATABASE_PATH)
       rescue
         destroy
       end

data/lib/ripe/version.rb

@@ -1,3 +1,3 @@
 module Ripe
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/ripe/worker_controller.rb

@@ -48,7 +48,13 @@ module Ripe
 
     def local(workers)
       distribute workers do |worker|
+        worker.update(status: :active_local)
+
         `bash #{worker.sh}`
+        exit_code = $?.to_i
+
+        worker.update(status:    :completed,
+                      exit_code: exit_code)
       end
     end
 
@@ -60,7 +66,7 @@ module Ripe
 
     def start(workers)
       distribute workers do |worker|
-        if worker.status == 'prepared'
+        if worker.prepared?
           worker.update(status: :queueing,
                         moab_id: `qsub '#{worker.sh}'`.strip.split(/\./).first)
         else

data/lib/ripe/worker_controller/preparer.rb

@@ -56,7 +56,7 @@ module Ripe
       #   and default params
 
       def load_workflow(workflow, params)
-        filename = Library.find(:workflow, workflow)
+        filename = Library.find(:workflow, "#{workflow}.rb")
         abort "Could not find workflow #{workflow}." if filename == nil
         require_relative filename
 
@@ -131,10 +131,12 @@ module Ripe
         File.open(worker.sh, 'w') { |f| f.write(worker_block.command) }
 
         worker.update({
-          status:   :prepared,
-          ppn:      params[:ppn],
-          queue:    params[:queue],
-          walltime: params[:walltime],
+          status:       :prepared,
+          ppn:          params[:ppn],
+          queue:        params[:queue],
+          walltime:     params[:walltime],
+          user:         `whoami`.chomp,
+          project_name: params[:project_name],
         })
 
         worker
@@ -163,8 +165,9 @@ module Ripe
                 block:  subblock.id,
               })
 
-              File.open(task.sh, 'w') { |f| f.write(subblock.command) }
               subblock.vars.merge!(log: task.log)
+              File.open(task.sh, 'w') { |f| f.write(subblock.command) }
+              subblock.vars
             else
               subblock.blocks.each(&post_var_assign)
             end

data/lib/ripe/worker_controller/syncer.rb

@@ -90,7 +90,7 @@ module Ripe
 
         @completed_workers = running_workers.select do |worker|
           !running_job_ids.include?(worker.moab_id) &&
-            worker.status != 'cancelled'
+            !worker.cancelled? && worker.user == `whoami`.chomp
         end
       end
 

data/spec/cli_spec.rb

@@ -11,4 +11,20 @@ describe CLI do
       expect(test_hash_opts).to eql ref_hash_opts
     end
   end
+
+  describe '::Helper#parse_config' do
+    it 'parses json file info symbolized hash' do
+      file = Tempfile.new("json")
+      begin
+        file.write("{\"a\": 1, \"b\": 2, \"c\": \"bob\"}")
+        file.rewind
+        test_hash_config = CLI::Helper.parse_config(file.path)
+        ref_hash_config = {:a=>1, :b=>2, :c=>"bob"}
+        expect(test_hash_config).to eql ref_hash_config
+      ensure
+        file.close
+      end
+
+    end
+  end
 end

data/spec/library_spec.rb

@@ -12,9 +12,9 @@ describe Library do
     end
 
     it 'cannot resolve components of the test library' do
-      expect(@library.find(:task, 'foo')).to eql nil
-      expect(@library.find(:task, 'bar')).to eql nil
-      expect(@library.find(:workflow, 'foobar')).to eql nil
+      expect(@library.find(:task, 'foo.sh')).to eql nil
+      expect(@library.find(:task, 'bar.sh')).to eql nil
+      expect(@library.find(:workflow, 'foobar.rb')).to eql nil
     end
   end
 
@@ -37,17 +37,17 @@ describe Library do
     end
 
     it 'resolves task components of the test library' do
-      expect(@library.find(:task, 'foo')).to eql @test.tasks['foo']
-      expect(@library.find(:task, 'bar')).to eql @test.tasks['bar']
+      expect(@library.find(:task, 'foo.sh')).to eql @test.tasks['foo']
+      expect(@library.find(:task, 'bar.sh')).to eql @test.tasks['bar']
     end
 
     it 'resolves workflows components of the test library' do
-      expect(@library.find(:workflow, 'foobar')).to eql @test.workflows['foobar']
+      expect(@library.find(:workflow, 'foobar.rb')).to eql @test.workflows['foobar']
     end
 
-    it 'cannot resolve non-existing componenets' do
-      expect(@library.find(:task, 'other')).to eql nil
-      expect(@library.find(:workflow, 'other')).to eql nil
+    it 'cannot resolve non-existing components' do
+      expect(@library.find(:task, 'other.sh')).to eql nil
+      expect(@library.find(:workflow, 'other.rb')).to eql nil
     end
   end
 end

data/spec/testpack.rb

@@ -8,6 +8,7 @@ module Ripe
       @tasks = {
         'foo' => "#{@lib_path}/tasks/foo.sh",
         'bar' => "#{@lib_path}/tasks/bar.sh",
+        'foo_erb' => "#{@lib_path}/tasks/foo_erb.sh"
       }
       @workflows = {
         'foobar' => "#{@lib_path}/workflows/foobar.rb",
@@ -20,7 +21,8 @@ module Ripe
       @steps = [
         'foo_input.txt',
         'foo_output.txt',
-        'bar_output.txt'
+        'bar_output.txt',
+        'foo_erb_output.txt'
       ]
     end
   end

data/spec/testpack/.ripe/tasks/foo.sh.erb

@@ -0,0 +1,3 @@
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "<%= vars.input_foo %>") <%= vars.foo_message %>" > "<%= vars.output_foo %>"

data/spec/testpack/.ripe/workers/1/1.sh

@@ -4,6 +4,7 @@
 INPUT_FOO="Sample1/foo_input.txt"
 FOO_MESSAGE="For You"
 OUTPUT_FOO="Sample1/foo_output.txt"
+LOG=".ripe/workers/1/1.log"
 
 exec 1>"$LOG" 2>&1
 

data/spec/testpack/.ripe/workers/1/2.sh

@@ -4,6 +4,7 @@
 INPUT_BAR="Sample1/foo_input.txt"
 BAR_MESSAGE="Bar"
 OUTPUT_BAR="Sample1/bar_output.txt"
+LOG=".ripe/workers/1/2.log"
 
 exec 1>"$LOG" 2>&1
 

data/spec/testpack/.ripe/workers/1/3.sh

@@ -0,0 +1,12 @@
+
+# <foo.sh.erb>
+
+exec 1>".ripe/workers/1/3.log" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "Sample1/foo_input.txt") For You" > "Sample1/foo_erb_output.txt"
+
+echo "##.DONE.##"
+
+# </foo.sh.erb>

data/spec/testpack/.ripe/workers/2/4.sh

@@ -1,16 +1,17 @@
 
-# <bar.sh>
+# <foo.sh>
 
-INPUT_BAR="Sample2/foo_input.txt"
-BAR_MESSAGE="Bar"
-OUTPUT_BAR="Sample2/bar_output.txt"
+INPUT_FOO="Sample2/foo_input.txt"
+FOO_MESSAGE="For You"
+OUTPUT_FOO="Sample2/foo_output.txt"
+LOG=".ripe/workers/2/4.log"
 
 exec 1>"$LOG" 2>&1
 
-# Bar is the most important consequence of Foo.
+# Foo is certainly one of the most important prerequisites to Bar.
 
-echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
 
 echo "##.DONE.##"
 
-# </bar.sh>
+# </foo.sh>

data/spec/testpack/.ripe/workers/2/5.sh

@@ -0,0 +1,17 @@
+
+# <bar.sh>
+
+INPUT_BAR="Sample2/foo_input.txt"
+BAR_MESSAGE="Bar"
+OUTPUT_BAR="Sample2/bar_output.txt"
+LOG=".ripe/workers/2/5.log"
+
+exec 1>"$LOG" 2>&1
+
+# Bar is the most important consequence of Foo.
+
+echo "$(cut -d' ' -f1 "$INPUT_BAR") $BAR_MESSAGE" > "$OUTPUT_BAR"
+
+echo "##.DONE.##"
+
+# </bar.sh>

data/spec/testpack/.ripe/workers/2/6.sh

@@ -0,0 +1,12 @@
+
+# <foo.sh.erb>
+
+exec 1>".ripe/workers/2/6.log" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "Sample2/foo_input.txt") For You" > "Sample2/foo_erb_output.txt"
+
+echo "##.DONE.##"
+
+# </foo.sh.erb>

data/spec/testpack/.ripe/workers/3/{5.sh → 7.sh}

@@ -4,6 +4,7 @@
 INPUT_FOO="Sample3/foo_input.txt"
 FOO_MESSAGE="For You"
 OUTPUT_FOO="Sample3/foo_output.txt"
+LOG=".ripe/workers/3/7.log"
 
 exec 1>"$LOG" 2>&1
 

data/spec/testpack/.ripe/workers/3/{6.sh → 8.sh}

@@ -4,6 +4,7 @@
 INPUT_BAR="Sample3/foo_input.txt"
 BAR_MESSAGE="Bar"
 OUTPUT_BAR="Sample3/bar_output.txt"
+LOG=".ripe/workers/3/8.log"
 
 exec 1>"$LOG" 2>&1
 

data/spec/testpack/.ripe/workers/3/9.sh

@@ -0,0 +1,12 @@
+
+# <foo.sh.erb>
+
+exec 1>".ripe/workers/3/9.log" 2>&1
+
+# Foo is certainly one of the most important prerequisites to Bar.
+
+echo "$(cat "Sample3/foo_input.txt") For You" > "Sample3/foo_erb_output.txt"
+
+echo "##.DONE.##"
+
+# </foo.sh.erb>

data/spec/testpack/.ripe/workflows/foobar.rb

@@ -18,6 +18,12 @@ workflow 'foobar' do
       param :output_bar,  "#{sample}/bar_output.txt"
     end
 
-    foo + bar
+    foo_erb = task 'foo', type: 'erb' do
+      param :input_foo,   "#{sample}/foo_input.txt"
+      param :foo_message, 'For You'
+      param :output_foo,  "#{sample}/foo_erb_output.txt"
+    end
+
+    foo + (bar | foo_erb)
   end
 end

data/spec/testpack/Sample1/foo_erb_output.txt

@@ -0,0 +1 @@
+One Foo For You

data/spec/testpack/Sample2/foo_erb_output.txt

@@ -0,0 +1 @@
+Two Foo For You

data/spec/testpack/Sample3/foo_erb_output.txt

@@ -0,0 +1 @@
+Three Foo For You

data/spec/worker_controller_spec.rb

@@ -6,7 +6,7 @@ require 'digest'
 require 'fileutils'
 
 def signature(file)
-  Digest::MD5.hexdigest(File.read(file))
+  Digest::MD5.hexdigest(File.read(file).gsub(/LOG=.*\n/, '').gsub(/exec 1>".*\n/, '') )
 end
 
 describe WorkerController do
@@ -64,8 +64,8 @@ describe WorkerController do
         ref_tasks = DB::Worker.find(1).tasks
         test_tasks = DB::Worker.find(4).tasks
 
-        expect(ref_tasks.length).to eql 2
-        expect(test_tasks.length).to eql 2
+        expect(ref_tasks.length).to eql 3
+        expect(test_tasks.length).to eql 3
 
         ref_tasks.zip(test_tasks).map do |ref, test|
           ref_hash = signature(ref.sh)
@@ -86,7 +86,7 @@ describe WorkerController do
         ref_tasks = DB::Worker.find(1).tasks
         test_tasks = DB::Worker.find(5).tasks
 
-        expect(ref_tasks.length).to eql 2
+        expect(ref_tasks.length).to eql 3
         expect(test_tasks.length).to eql 1
 
         ref_hash = signature(ref_tasks.first.sh)
@@ -107,8 +107,8 @@ describe WorkerController do
         ref_tasks = DB::Worker.find(2).tasks
         test_tasks = DB::Worker.find(6).tasks
 
-        expect(ref_tasks.length).to eql 2
-        expect(test_tasks.length).to eql 2
+        expect(ref_tasks.length).to eql 3
+        expect(test_tasks.length).to eql 3
 
         ref_tasks.zip(test_tasks).map do |ref, test|
           ref_hash = signature(ref.sh)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ripe
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Nicolas De Jay
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2017-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -226,7 +226,9 @@ files:
 - bin/ripe
 - lib/ripe.rb
 - lib/ripe/blocks.rb
+- lib/ripe/blocks/bash_block.rb
 - lib/ripe/blocks/block.rb
+- lib/ripe/blocks/erb_block.rb
 - lib/ripe/blocks/liquid_block.rb
 - lib/ripe/blocks/multi_block.rb
 - lib/ripe/blocks/parallel_block.rb
@@ -257,23 +259,30 @@ files:
 - spec/testpack/.ripe/meta.db
 - spec/testpack/.ripe/tasks/bar.sh
 - spec/testpack/.ripe/tasks/foo.sh
+- spec/testpack/.ripe/tasks/foo.sh.erb
 - spec/testpack/.ripe/workers/1/1.sh
 - spec/testpack/.ripe/workers/1/2.sh
+- spec/testpack/.ripe/workers/1/3.sh
 - spec/testpack/.ripe/workers/1/job.sh
-- spec/testpack/.ripe/workers/2/3.sh
 - spec/testpack/.ripe/workers/2/4.sh
+- spec/testpack/.ripe/workers/2/5.sh
+- spec/testpack/.ripe/workers/2/6.sh
 - spec/testpack/.ripe/workers/2/job.sh
-- spec/testpack/.ripe/workers/3/5.sh
-- spec/testpack/.ripe/workers/3/6.sh
+- spec/testpack/.ripe/workers/3/7.sh
+- spec/testpack/.ripe/workers/3/8.sh
+- spec/testpack/.ripe/workers/3/9.sh
 - spec/testpack/.ripe/workers/3/job.sh
 - spec/testpack/.ripe/workflows/foobar.rb
 - spec/testpack/Sample1/bar_output.txt
+- spec/testpack/Sample1/foo_erb_output.txt
 - spec/testpack/Sample1/foo_input.txt
 - spec/testpack/Sample1/foo_output.txt
 - spec/testpack/Sample2/bar_output.txt
+- spec/testpack/Sample2/foo_erb_output.txt
 - spec/testpack/Sample2/foo_input.txt
 - spec/testpack/Sample2/foo_output.txt
 - spec/testpack/Sample3/bar_output.txt
+- spec/testpack/Sample3/foo_erb_output.txt
 - spec/testpack/Sample3/foo_input.txt
 - spec/testpack/Sample3/foo_output.txt
 - spec/worker_controller_spec.rb
@@ -297,7 +306,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Abstraction layer between the MOAB/Torque stack and your pipeline.
@@ -309,24 +318,30 @@ test_files:
 - spec/testpack/.ripe/meta.db
 - spec/testpack/.ripe/tasks/bar.sh
 - spec/testpack/.ripe/tasks/foo.sh
+- spec/testpack/.ripe/tasks/foo.sh.erb
 - spec/testpack/.ripe/workers/1/1.sh
 - spec/testpack/.ripe/workers/1/2.sh
+- spec/testpack/.ripe/workers/1/3.sh
 - spec/testpack/.ripe/workers/1/job.sh
-- spec/testpack/.ripe/workers/2/3.sh
 - spec/testpack/.ripe/workers/2/4.sh
+- spec/testpack/.ripe/workers/2/5.sh
+- spec/testpack/.ripe/workers/2/6.sh
 - spec/testpack/.ripe/workers/2/job.sh
-- spec/testpack/.ripe/workers/3/5.sh
-- spec/testpack/.ripe/workers/3/6.sh
+- spec/testpack/.ripe/workers/3/7.sh
+- spec/testpack/.ripe/workers/3/8.sh
+- spec/testpack/.ripe/workers/3/9.sh
 - spec/testpack/.ripe/workers/3/job.sh
 - spec/testpack/.ripe/workflows/foobar.rb
 - spec/testpack/Sample1/bar_output.txt
+- spec/testpack/Sample1/foo_erb_output.txt
 - spec/testpack/Sample1/foo_input.txt
 - spec/testpack/Sample1/foo_output.txt
 - spec/testpack/Sample2/bar_output.txt
+- spec/testpack/Sample2/foo_erb_output.txt
 - spec/testpack/Sample2/foo_input.txt
 - spec/testpack/Sample2/foo_output.txt
 - spec/testpack/Sample3/bar_output.txt
+- spec/testpack/Sample3/foo_erb_output.txt
 - spec/testpack/Sample3/foo_input.txt
 - spec/testpack/Sample3/foo_output.txt
 - spec/worker_controller_spec.rb
-has_rdoc: 

data/spec/testpack/.ripe/workers/2/3.sh

@@ -1,16 +0,0 @@
-
-# <foo.sh>
-
-INPUT_FOO="Sample2/foo_input.txt"
-FOO_MESSAGE="For You"
-OUTPUT_FOO="Sample2/foo_output.txt"
-
-exec 1>"$LOG" 2>&1
-
-# Foo is certainly one of the most important prerequisites to Bar.
-
-echo "$(cat "$INPUT_FOO") $FOO_MESSAGE" > "$OUTPUT_FOO"
-
-echo "##.DONE.##"
-
-# </foo.sh>