rbbt-util 5.33.7 → 5.33.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4272048a5c86e74b051600f0db895ea42f4ad51509a0de9ce88452d27133746
-  data.tar.gz: f10d19e028de390beb5e2649c7d2f1e0a7754d3235abe3eeba2790946b347f64
+  metadata.gz: e5c26a161d8529f19d32ace8b5565042b4baa6fef3d69580f46aadc523e8cc51
+  data.tar.gz: 8ee1305d3a66a55273474da60e84cec3385391ad99c8c87e764fd72eaaf29285
 SHA512:
-  metadata.gz: 0667eb9d5783d7722c33f72a66a1c406d330d086a37660aa7682006d10fe15860a9e4cbfcb461f17ed15b146fa294a6f6d8158d937884f14dc7952908f63c357
-  data.tar.gz: f3ba25b347bc4a7f7b56ecdbd1a0bef932b646ac030b3d98fe6ddb76f36a6863073f4692747ceef32c42e4344538a7ee19b531a25b16cc2fa208b201887b441e
+  metadata.gz: 6b9ddf9ff821584ef21f7c30be381965ea9a8cf1cce7e9913cc11c118fa1d31211fe321d72c5d8daabcb14aac5b20df45c527fefa1d142c7a08633c3818b79dd
+  data.tar.gz: c0ab742b23fe87fd7f52bfdfe75690a02566547a34a484912c74766cff70c83634434d5a67a9e665bfdffb28be143afdb20028356befe045e19b14bcc83a21ff

data/lib/rbbt/hpc/batch.rb

@@ -88,8 +88,8 @@ module HPC
     def rbbt_job_exec_cmd(job, options)
 
       jobname  = job.clean_name
-      workflow = job.workflow
-      task     = job.task_name
+      workflow = job.original_workflow || job.workflow
+      task     = job.original_task_name || job.task_name
 
       Misc.add_defaults options, :jobname => jobname
 
@@ -535,8 +535,8 @@ env > #{batch_options[:fenv]}
         :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
         :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
 
-      workflow = job.workflow
-      task_name = job.task_name
+      workflow = job.original_workflow ||job.workflow
+      task_name = job.original_task_name || job.task_name
 
       workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
 

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -101,7 +101,9 @@ module HPC
 
       batches.each do |batch|
         job_rules = batch[:jobs].inject(nil) do |acc,job|
-          task_rules = task_specific_rules(rules, job.workflow, job.task_name)
+          workflow = job.original_workflow || job.workflow
+          task_name = job.original_task_name || job.task_name
+          task_rules = task_specific_rules(rules, workflow, task_name)
           acc = accumulate_rules(acc, task_rules.dup)
         end
 
@@ -120,13 +122,16 @@ module HPC
         end
 
         batches.each do |batch|
+          next if batch[:top_level].overriden?
           next unless batch[:rules][:skip]
           batch[:rules].delete :skip
           next if batch[:deps].nil?
 
           if batch[:deps].any?
+            batch_dep_jobs = batch[:top_level].rec_dependencies
             target = batch[:deps].select do |target|
-              (batch[:deps] - [target] - target[:deps]).empty?
+              batch_dep_jobs.include?(target[:top_level]) && # Don't piggyback batches that are an input dependency, only real dependencies
+                (batch[:deps] - [target] - target[:deps]).empty?
             end.first
             next if target.nil?
             target[:jobs] = batch[:jobs] + target[:jobs]

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -4,8 +4,10 @@ module HPC
       return [] if Symbol === job.overriden
       matches = []
       chains.each do |name, chain|
-        next unless chain[:tasks].include?(job.workflow.to_s)
-        next unless chain[:tasks][job.workflow.to_s].include?(job.task_name.to_s)
+        workflow = job.original_workflow || job.workflow
+        task_name = job.original_task_name || job.task_name
+        next unless chain[:tasks].include?(workflow.to_s)
+        next unless chain[:tasks][workflow.to_s].include?(task_name.to_s)
         matches << name
       end
       matches
@@ -38,46 +40,50 @@ module HPC
     end
 
     def self.job_chains(rules, job)
-      chains = self.parse_chains(rules)
-
-      matches = check_chains(chains, job)
-
-      dependencies = job_dependencies(job)
-
-      job_chains = []
-      new_job_chains = {}
-      dependencies.each do |dep|
-        dep_matches = check_chains(chains, dep)
-        common = matches & dep_matches
-
-        dep_chains = job_chains(rules, dep)
-        found = []
-        dep_chains.each do |match,info|
-          if common.include?(match)
-            found << match
-            new_info = new_job_chains[match] ||= {}
-            new_info[:jobs] ||= []
-            new_info[:jobs].concat info[:jobs]
-            new_info[:top_level] = job
-          else
+      @@job_chains ||= {}
+      @@job_chains[Misc.digest([rules, job.path].inspect)] ||= 
+        begin
+          chains = self.parse_chains(rules)
+
+          matches = check_chains(chains, job)
+
+          dependencies = job_dependencies(job)
+
+          job_chains = []
+          new_job_chains = {}
+          dependencies.each do |dep|
+            dep_matches = check_chains(chains, dep)
+            common = matches & dep_matches
+
+            dep_chains = job_chains(rules, dep)
+            found = []
+            dep_chains.each do |match,info|
+              if common.include?(match)
+                found << match
+                new_info = new_job_chains[match] ||= {}
+                new_info[:jobs] ||= []
+                new_info[:jobs].concat info[:jobs]
+                new_info[:top_level] = job
+              else
+                job_chains << [match, info]
+              end
+            end
+
+            (common - found).each do |match|
+              info = {}
+              info[:jobs] = [job, dep]
+              info[:top_level] = job
+              job_chains << [match, info]
+            end
+          end
+
+          new_job_chains.each do |match,info|
+            info[:jobs].prepend job
             job_chains << [match, info]
           end
-        end
 
-        (common - found).each do |match|
-          info = {}
-          info[:jobs] = [job, dep]
-          info[:top_level] = job
-          job_chains << [match, info]
+          job_chains
         end
-      end
-
-      new_job_chains.each do |match,info|
-        info[:jobs].prepend job
-        job_chains << [match, info]
-      end
-
-      job_chains
     end
 
   end

data/lib/rbbt/hpc/orchestrate.rb

@@ -13,7 +13,11 @@ module HPC
       all_deps = rec_dependencies + [job]
 
       all_deps.each do |dep|
-        Step.prepare_for_execution(dep)
+        begin
+          Step.prepare_for_execution(dep)
+        rescue RbbtException
+          next
+        end
       end
 
     end
@@ -26,6 +30,7 @@ module HPC
       options.delete "printpath"
       options.delete "detach"
       options.delete "jobname"
+      options.delete "load_inputs"
 
       Log.high "Prepare for exec"
       prepare_for_execution(job)
@@ -40,8 +45,8 @@ module HPC
 
       IndiferentHash.setup(rules)
 
-      Log.high "Compute batches"
       batches = HPC::Orchestration.job_batches(rules, job)
+      Log.high "Compute #{batches.length} batches"
 
       batch_ids = {}
       while batches.any?
@@ -77,9 +82,9 @@ module HPC
 
         if options[:dry_run]
           puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-          puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
           puts Log.color(:yellow, "Path: ") + top[:top_level].path
-          puts Log.color(:yellow, "Options: ") + Misc.fingerprint(job_options)
+          puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature
         else
           id = run_job(top[:top_level], job_options)

data/lib/rbbt/hpc/slurm.rb

@@ -94,8 +94,8 @@ export BATCH_SYSTEM=SLURM
 
       return if Open.exists?(fexit)
 
-      STDERR.puts Log.color(:magenta, "Issuing SLURM file: #{fcmd}")
-      STDERR.puts Open.read(fcmd)
+      Log.info "Issuing SLURM file: #{fcmd}"
+      Log.debug Open.read(fcmd)
 
       if File.exists?(fjob)
         job = Open.read(fjob).to_i

data/lib/rbbt/resource/path.rb

@@ -167,8 +167,7 @@ module Path
 
     @path ||= {}
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
-    key_elems = [where, caller_lib, rsearch_paths, paths]
-    key = Misc.obj2digest(key_elems.inspect)
+    key = [where, caller_lib, rsearch_paths, paths].inspect
     self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
 
     return @path[key] if @path[key]

data/lib/rbbt/resource/util.rb

@@ -1,13 +1,26 @@
 module Path
 
   def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
-    file = caller.reject{|l| 
-      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
-      l =~ /rbbt\/resource\/path\.rb/ or
-      l =~ /rbbt\/persist.rb/ or
-      l =~ /rbbt\/util\/misc\.rb/ or
-      l =~ /progress-monitor\.rb/ 
-    }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
+    #file = caller.reject{|l| 
+    #  l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+    #  l =~ /rbbt\/resource\/path\.rb/ or
+    #  l =~ /rbbt\/persist.rb/ or
+    #  l =~ /rbbt\/util\/misc\.rb/ or
+    #  l =~ /progress-monitor\.rb/ 
+    #}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
+    
+    
+    if file.nil?
+      caller_dup = caller.dup
+      while file = caller_dup.shift
+        break unless file =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+          file =~ /rbbt\/resource\/path\.rb/ or
+          file =~ /rbbt\/persist.rb/ or
+          file =~ /rbbt\/util\/misc\.rb/ or
+          file =~ /progress-monitor\.rb/ 
+      end
+      file = file.sub(/\.rb[^\w].*/,'.rb')
+    end
 
     relative_to = [relative_to] unless Array === relative_to
     file = File.expand_path(file)

data/lib/rbbt/util/misc/inspect.rb

@@ -80,7 +80,7 @@ module Misc
       end
     when (defined? TSV and TSV)
       obj.with_unnamed do
-        "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
+        "TSV:{"<< fingerprint(obj.all_fields|| []) << ";" << fingerprint(obj.keys) << "}"
       end
     when Hash
       if obj.length > 10
@@ -297,14 +297,14 @@ module Misc
     str = case obj
           when nil
             'nil'
+          when Symbol 
+            obj.to_s
           when TrueClass
             'true'
           when FalseClass
             'false'
           when Hash
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
-          when Symbol 
-            obj.to_s
           when (defined?(Path) and Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
@@ -324,9 +324,10 @@ module Misc
             end
           when String
             good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
-            is_path = Path === obj
-            if good_filename && ! is_path
-              obj2str Path.setup(obj.dup)
+            if good_filename 
+              obj = obj.dup
+              obj.extend Path
+              obj2str obj
             else
               obj = obj.chomp if String === obj
               if obj.length > HASH2MD5_MAX_STRING_LENGTH
@@ -339,7 +340,7 @@ module Misc
             if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
               "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
             else
-              "[" << obj.collect{|v| obj2str(v)} * "," << "]"
+              "[" << obj.collect{|v| obj2str(v) } * "," << "]"
             end
           when TSV::Parser
             remove_long_items(obj)
@@ -354,7 +355,7 @@ module Misc
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
           when (defined? Step and Step)
-            "<IO:" << obj.short_path << ">"
+            "<IO:" << obj.short_path_real << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
               if defined?(Step) && Open.exists?(Step.info_file(obj.filename))

data/lib/rbbt/util/misc/system.rb

@@ -113,9 +113,10 @@ end
   end
 
   def self.is_filename?(string, need_to_exists = true)
+    return false if string.nil?
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
+    return true if String === string and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
     return false
   end
 

data/lib/rbbt/workflow/dependencies.rb

@@ -72,9 +72,12 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
+
+    dep.original_workflow ||= dep.workflow if dep.workflow
+    dep.original_task_name ||= dep.task_name if dep.workflow
+
     dep.workflow = workflow
     dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name if dep.workflow
 
 
     begin

data/lib/rbbt/workflow/step/accessor.rb

@@ -96,6 +96,10 @@ class Step
     [task_name, name] * "/"
   end
 
+  def short_path_real
+    [(Symbol === overriden ? overriden : task_name).to_s, name] * "/"
+  end
+
   def workflow_short_path
     return short_path unless workflow
     workflow.to_s + "#" + short_path

data/lib/rbbt/workflow/step/dependencies.rb

@@ -510,7 +510,7 @@ class Step
 
   def overriden?
     return true if @overriden
-    return true if dependencies.select{|dep| dep.overriden? }.any?
+    return true if dependencies && dependencies.select{|dep| dep.overriden? }.any?
     info[:archived_info].each do |f,i|
       next if Symbol === i
       return true if i[:overriden] || i["overriden"]

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -1,4 +1,5 @@
 module Workflow
+
   def self.load_inputs(dir, input_names, input_types)
     inputs = {}
     if File.exists?(dir) && ! File.directory?(dir)
@@ -20,21 +21,59 @@ module Workflow
         Log.debug "Trying #{ input }: #{file}"
         next unless file and (File.symlink?(file) || file.exists?)
 
-        type = input_types[input]
+        type = orig_type = input_types[input]
 
         type = :io if file.split(".").last == 'as_io'
 
+        type = :io_array if file.split(".").last == 'as_io_array'
+
+        type = :step if file.split(".").last == 'as_step'
+
+        type = :step_array if file.split(".").last == 'as_step_array'
+
+        type = :step_file if file.split(".").last == 'as_step_file'
+
+        type = :step_file_array if file.split(".").last == 'as_step_file_array'
+
         type = :path if file.split(".").last == 'as_path'
 
+        type = :path_array if file.split(".").last == 'as_path_array'
+
+        type = :filename if file.split(".").last == 'as_filename'
+
         type = :nofile if file.split(".").last == 'nofile'
 
         case type
         when :nofile
           inputs[input.to_sym]  = Open.realpath(file)
+        when :path_array
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n")
         when :path
-          inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file)).split("\n")
         when :io
-          inputs[input.to_sym] = Open.open(Open.realpath(file))
+          inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
+        when :step_array
+          steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
+          inputs[input.to_sym] = steps
+        when :step
+          steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
+          inputs[input.to_sym] = steps.first
+        when :step_file
+          path = Open.read(file).strip
+          path.extend Path
+          step_path = path.match(/(.*)\.files/)[1]
+          path.resource = Step.new step_path
+          inputs[input.to_sym] = path
+        when :step_file_array
+          paths = Open.read(file).split("\n")
+          paths.each do |path| 
+            path.extend Path
+            step_path = path.match(/(.*)\.files/)[1]
+            path.resource = Step.new step_path
+          end
+          inputs[input.to_sym] = paths
         when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
@@ -87,60 +126,149 @@ module Workflow
     job(task_name, jobname, inputs)
   end
 
+  #def self.load_inputs_old(dir, input_names, input_types)
+  #  inputs = {}
+  #  if File.exists?(dir) && ! File.directory?(dir)
+  #    Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+  #    tarfile = dir
+  #    digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+  #    tmpdir = Rbbt.tmp.input_bundle[digest].find
+  #    Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+  #    files = tmpdir.glob("*")
+  #    if files.length == 1 && File.directory?(files.first)
+  #      tmpdir = files.first
+  #    end
+  #    load_inputs(tmpdir, input_names, input_types)
+  #  else
+  #    dir = Path.setup(dir.dup)
+  #    input_names.each do |input|
+  #      file = dir[input].find
+  #      file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+  #      Log.debug "Trying #{ input }: #{file}"
+  #      next unless file and (File.symlink?(file) || file.exists?)
+
+  #      type = input_types[input]
+
+  #      type = :io if file.split(".").last == 'as_io'
+
+  #      type = :path if file.split(".").last == 'as_path'
+
+  #      type = :filename if file.split(".").last == 'as_filename'
+
+  #      type = :nofile if file.split(".").last == 'nofile'
+
+  #      case type
+  #      when :nofile
+  #        inputs[input.to_sym]  = Open.realpath(file)
+  #      when :path
+  #        inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+  #      when :io
+  #        inputs[input.to_sym] = Open.open(Open.realpath(file))
+  #      when :file, :binary
+  #        Log.debug "Pointing #{ input } to #{file}"
+  #        if file =~ /\.yaml/
+  #          inputs[input.to_sym]  = YAML.load(Open.read(file))
+  #        else
+  #          if File.symlink?(file)
+  #            link_target = File.expand_path(File.readlink(file), File.dirname(file))
+  #            inputs[input.to_sym]  = link_target
+  #          else
+  #            inputs[input.to_sym]  = Open.realpath(file)
+  #          end
+  #        end
+  #      when :text
+  #        Log.debug "Reading #{ input } from #{file}"
+  #        inputs[input.to_sym]  = Open.read(file)
+  #      when :array
+  #        Log.debug "Reading array #{ input } from #{file}"
+  #        inputs[input.to_sym]  = Open.read(file).split("\n")
+  #      when :tsv
+  #        Log.debug "Opening tsv #{ input } from #{file}"
+  #        inputs[input.to_sym]  = TSV.open(file)
+  #      when :boolean
+  #        inputs[input.to_sym]  = (file.read.strip == 'true')
+  #      else
+  #        Log.debug "Loading #{ input } from #{file}"
+  #        inputs[input.to_sym]  = file.read.strip
+  #      end
+
+  #    end
+  #    inputs = IndiferentHash.setup(inputs)
+
+  #    dir.glob("*#*").each do |od|
+  #      name = File.basename(od)
+  #      value = Open.read(od)
+  #      Log.debug "Loading override dependency #{ name } as #{value}"
+  #      inputs[name] = value.chomp
+  #    end
+
+  #    inputs
+  #  end
+  #end
 end
 
 class Step
-  def self.save_inputs(inputs, input_types, input_options, dir)
-    inputs.each do |name,value|
-      type = input_types[name]
-      type = type.to_s if type
-      path = File.join(dir, name.to_s)
-
-      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
-      Log.debug "Saving job input #{name} (#{type}) into #{path}"
-
-      case
-      when IO === value
-        Open.write(path, value.to_s)
-      when Step === value
-        Open.ln_s(value.path, path)
-      when type.to_s == "binary"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        elsif String === value && Misc.is_filename?(value, false)
-          Open.write(path + '.as_path' , value)
-        else
-          Open.write(path, value, :mode => 'wb')
-        end
-      when type.to_s == "file"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
+  def self.save_input(name, value, type, dir)
+    path = File.join(dir, name.to_s)
+
+    case value
+    when Path
+      if Step === value.resource
+        path = path + '.as_step_file'
+      else
+        path = path + '.as_path'
+      end
+    when String
+      if Misc.is_filename?(value, false)
+        value = value.dup
+        value.extend Path
+        return save_input(name, value, type, dir)
+      end
+    when IO
+      path = path + '.as_io'
+    when Step
+      value = value.path
+      path = path + '.as_step'
+    when Array
+      case value.first
+      when Path
+        if Step === value.first.resource
+          path = path + '.as_step_file_array'
         else
-          value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
-          value = "#{value}" if Path === value
-          Open.write(path + '.yaml', value.to_yaml)
+          path = path + '.as_path_array'
         end
-      when Array === value
-        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
-      when IO === value
-        if value.filename && String === value.filename && File.exists?(value.filename)
-          Open.ln_s(value.filename, path)
-        else
-          Open.write(path, value)
+      when String
+        if Misc.is_filename?(value.first, false)
+          path = path + '.as_path_array'
         end
-      else
-        Open.write(path, value.to_s)
+      when IO
+        path = path + '.as_io_array'
+      when Step
+        path = path + '.as_step_array'
+        value = value.collect{|s| s.path }
       end
+
+      value = value * "\n"
+    end
+
+    Log.debug "Saving job input #{name} (#{type}) into #{path}"
+    Open.write(path, value.to_s)
+  end
+
+  def self.save_inputs(inputs, input_types, input_options, dir)
+    inputs.each do |name,value|
+      type = input_types[name]
+      type = type.to_s if type
+
+      save_input(name, value, type, dir)
     end.any?
   end
 
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = Symbol === job.overriden ? job.overriden : job.task_name
-    workflow = job.workflow
+    task_name = job.original_task_name || job.task_name
+    workflow = job.original_workflow || job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     if workflow
       task_info = IndiferentHash.setup(workflow.task_info(task_name))
@@ -176,4 +304,60 @@ class Step
 
     inputs.keys
   end
+
+  #def self.save_inputs_old(inputs, input_types, input_options, dir)
+  #  inputs.each do |name,value|
+  #    type = input_types[name]
+  #    type = type.to_s if type
+  #    path = File.join(dir, name.to_s)
+
+  #    if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+  #      path = path + '.as_io' 
+  #    elsif Misc.is_filename?(value, false)
+  #      path = path + '.as_filename'
+  #    end
+
+  #    Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+  #    case
+  #    when IO === value
+  #      Open.write(path, value.to_s)
+  #    when Step === value
+  #      Open.ln_s(value.path, path)
+  #    when type.to_s == "binary"
+  #      if String === value && File.exists?(value)
+  #        value = File.expand_path(value)
+  #        Open.ln_s(value, path)
+  #      elsif String === value && Misc.is_filename?(value, false)
+  #        Open.write(path + '.as_path' , value)
+  #      else
+  #        Open.write(path, value, :mode => 'wb')
+  #      end
+  #    when TSV === value
+  #      Open.write(path, value.to_s)
+  #    when Array === value
+  #      Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+  #    when %w(file tsv array).include?(type.to_s)
+  #      if String === value && File.exists?(value)
+  #        value = File.expand_path(value)
+  #        Open.ln_s(value, path)
+  #      elsif String === value && Misc.is_filename?(value, false)
+  #        Open.write(path + '.as_path' , value)
+  #      else
+  #        value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
+  #        value = "#{value}" if Path === value
+  #        Open.write(path + '.yaml', value.to_yaml)
+  #      end
+  #    when IO === value
+  #      if value.filename && String === value.filename && File.exists?(value.filename)
+  #        Open.ln_s(value.filename, path)
+  #      else
+  #        Open.write(path, value)
+  #      end
+  #    else
+  #      Open.write(path, value.to_s)
+  #    end
+  #  end.any?
+  #end
+
 end

data/lib/rbbt/workflow/step.rb

@@ -15,7 +15,7 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
-  attr_accessor :real_inputs, :original_task_name
+  attr_accessor :real_inputs, :original_task_name, :original_workflow
 
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
@@ -376,9 +376,9 @@ class Step
                        }
                        raise "Dependency step not found: #{ name }" if deps.empty?
                        if (deps & self.dependencies).any?
-                         (deps & self.dependencies).first
+                         (deps & self.dependencies).last
                        else
-                         deps.first
+                         deps.last
                        end
                      end
   end

data/share/install/software/lib/install_helpers

@@ -185,6 +185,10 @@ compile(){
     local name=$1; shift
     local extra="$@"
 
+    if [ -f bootstrap ]; then
+      ./bootstrap
+    fi
+
     if [ -f Makefile -o -f makefile ]; then
         make -j 4 || exit -1
         make install || echo "No install"

data/share/rbbt_commands/hpc/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -185,6 +186,42 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +269,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -305,12 +342,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/hpc/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/lsf/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -185,6 +186,42 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +269,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -305,12 +342,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/lsf/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/list

@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
 -l--long Show more entries
+-c--compressed Show compressed information about entries
 -p--progress Report progress of job and the dependencies
 -BP--batch_parameters show batch parameters
 -BPP--batch_procpath show Procpath performance summary
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -185,6 +186,42 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
 
+ count += 1
+
+ if options[:compressed] 
+   status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+   if different_system
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+   else
+     status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
+   end 
+   prog_rep = []
+   if options[:progress] 
+     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
+     if step_line
+       require 'rbbt/workflow'
+       step_path = step_line.split(": ").last.strip
+       step = Step.new step_path
+       step.load_dependencies_from_info
+       has_bar = false
+       (step.rec_dependencies + [step]).reverse.each do |j|
+         next if j.done? 
+         if j.file(:progress).exists?
+           bar = Log::ProgressBar.new 
+           bar.load(j.file(:progress).yaml)
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
+         end
+       end
+     end
+   end
+   workflow, task, name = step_path.split("/")[-3..-1]
+   job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
+   puts [job_str, status, prog_rep ].flatten * " "
+   next
+ end
+
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +269,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    cpu_average = {}
    rss_average = {}
    perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
-    time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
-    time = time.to_f
-
-    cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
-    cpu_average[stat_pid] ||= {}
-    cpu_average[stat_pid][time] ||= []
-    cpu_average[stat_pid][time] << cpu.to_f
-    rss_average[time] ||= []
-    rss_average[time] << rss.to_f * page_size
+     time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
+     time = time.to_f
+
+     cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
+     cpu_average[stat_pid] ||= {}
+     cpu_average[stat_pid][time] ||= []
+     cpu_average[stat_pid][time] << cpu.to_f
+     rss_average[time] ||= []
+     rss_average[time] << rss.to_f * page_size
    end
 
    ticks = 0
    cpu_average.each do |stat_pid, cpu_average_pid|
-    start = cpu_average_pid.keys.sort.first
-    eend = cpu_average_pid.keys.sort.last
-    ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
+     start = cpu_average_pid.keys.sort.first
+     eend = cpu_average_pid.keys.sort.last
+     ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
    end
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
@@ -305,12 +342,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 
- count += 1
-
 end
 
 puts 

data/share/rbbt_commands/slurm/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/workflow/task

@@ -76,6 +76,16 @@ def fix_options(workflow, task, job_options)
     type = input_types[name]
     type = type.to_sym if type
 
+    if Step === value
+        job_options_cleaned[name] = value
+        next
+    end
+
+    if Path === value && Step === value.resource
+        job_options_cleaned[name] = value
+        next
+    end
+
     value = case type
             when nil
                 value
@@ -148,6 +158,7 @@ def fix_options(workflow, task, job_options)
             else
               value
             end
+
     job_options_cleaned[name] = value
   end
 

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -26,11 +26,26 @@ module TestSaveLoadWF
   task :prefix => :array do
     step(:reverse).run.collect{|e| "A-#{e}" }
   end
+
+  input :integer, :integer
+  input :float, :float
+  input :file, :file
+  input :file_no_file, :file, "", nil, :nofile => true
+  input :string, :string
+  input :select, :select
+  input :array, :array
+  input :array_no_file, :array, "", nil, :nofile => true
+  input :tsv, :tsv, "", nil, :nofile => true
+  input :tsv_no_file, :tsv, "", nil, :nofile => true
+  input :binary, :binary, "", nil, :nofile => true
+  input :binary_no_file, :tsv, "", nil, :nofile => true
+  task :save_test => :string do
+    "DONE"
+  end
 end
 
 class TestSaveLoad < Test::Unit::TestCase
   def test_save
-    Log.with_severity 0 do
     job = TestSaveLoadWF.job(:prefix)
     job.recursive_clean
     job = TestSaveLoadWF.job(:prefix)
@@ -40,6 +55,81 @@ class TestSaveLoad < Test::Unit::TestCase
       newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
       assert_equal job.rec_dependencies.last.path, newjob.path
     end
+  end
+
+  def test_save_all_normal
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    options = {
+      :float => 0.5,
+      :integer => 15,
+      :string => "STRING",
+      :select => "option",
+      :array => [0,1,2,3],
+      :tsv => tsv,
+    }
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        job = TestSaveLoadWF.job(:save_test, nil, options)
+        TmpFile.with_file do |directory|
+          Step.save_job_inputs(job, directory)
+          newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+          assert_equal job.path, newjob.path
+        end
+      end
+    end
+  end
+
+  def test_save_all_file
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        TmpFile.with_file do |input_file|
+          Path.setup(input_file)
+          options = {
+            :float => input_file.float,
+            :integer => input_file.integer,
+            :string => input_file.string,
+            :select => input_file.select,
+            :array => input_file.array,
+            :tsv => input_file.tsv_file,
+          }
+          job = TestSaveLoadWF.job(:save_test, nil, options)
+          TmpFile.with_file do |directory|
+            Step.save_job_inputs(job, directory)
+            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+            assert_equal job.path, newjob.path
+          end
+        end
+      end
+    end
+  end
+
+  def test_save_all_job_file
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        TmpFile.with_file do |input_file|
+          Path.setup(input_file)
+          options = {
+            :float => input_file.float,
+            :integer => input_file.integer,
+            :string => input_file.string,
+            :select => input_file.select,
+            :array => TestSaveLoadWF.job(:list),
+            :binary => TestSaveLoadWF.job(:list).file('binary'),
+            :tsv => input_file.tsv_file,
+          }
+          job = TestSaveLoadWF.job(:save_test, nil, options)
+          TmpFile.with_file do |directory|
+            Step.save_job_inputs(job, directory)
+            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+            assert_equal job.path, newjob.path
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.7
+  version: 5.33.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-10 00:00:00.000000000 Z
+date: 2022-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake