rbbt-util 5.32.23 → 5.32.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 900a7061d40ec2cbffd38886ef9629aa0c0ed6d6ee1aef6e0c95c51656230ef0
-  data.tar.gz: 13b965a0ec134a932c6ae0ae6a109fbc2bc83b7ffd855a1ef05ed4e1e51b9d88
+  metadata.gz: ffa357551766aa6ae6da35c586949c0ff05814171304ca5a874689418c7c48e2
+  data.tar.gz: 5e1e419fcf846fc8ead12a77e6713f198a74d516c0262c34b4d86380a58af092
 SHA512:
-  metadata.gz: 3b890e2b6fb3675df4319d0541d9057a54b0cb37286a814822915f5b5b061d4f7e02deab0afdd7cd0833cb741aeded7454a9585f93ff3486e595b99db4da00f6
-  data.tar.gz: 105072ca6c7b5c43c6a60455eaa9c7167feeb93f505c61c6aeffe6305de3166a6cb440c262abf025407306ba82c2fe3167f5f9e743e086d99fa936e44ea8e448
+  metadata.gz: a6afd86e7698e5b7e4588412ed678497c825242616bc75d2fa2ab835bc325a8a58826da164dea16cd5b4bbf81697ac5201bb347af220bab5ce238f8e0ce4edcb
+  data.tar.gz: 3e86fe7c2d84cc7a04cdd348f03a6323b5467723182057ec028b5d611b3dd200fce41949f1502ca59b0a295b5664f1d6ba259d27a196039f873261e9faf7a1a6

data/bin/rbbt

@@ -262,9 +262,8 @@ rescue ParameterException
   puts
   exit_status = -1
   exit exit_status
-rescue SystemExit
-rescue CmdStop
-  exit_status = $!.exit_status
+rescue SystemExit,CmdStop
+  exit_status = $!.status
   exit exit_status
 rescue Exception
   Log.exception $!

data/bin/rbbt_find.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find a path
+
+$ #{$0} [options] [<subpath>] <path>
+
+Use - to read from STDIN
+
+-h--help Print this help
+-w--workflows Workflow to load
+-s--search_path* Workflow to load
+-l--list List contents of resolved directories
+-n--nocolor Don't color output
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+subpath, path = ARGV
+path, subpath = subpath, nil if path.nil?
+
+begin
+  require 'rbbt/workflow'
+  workflow = Workflow.require_workflow subpath
+  subpath = workflow.libdir
+rescue
+  Log.exception $!
+end if subpath && subpath =~ /^[A-Z][a-zA-Z]+$/
+
+path = subpath ? Path.setup(subpath)[path] : Path.setup(path)
+
+search_path = options[:search_path].to_sym if options.include? :search_path
+nocolor = options[:nocolor]
+
+found = if search_path
+          [path.find(search_path)]
+        else
+          path.find_all
+        end
+
+found.each do |path|
+  if options[:list] && File.directory?(path)
+    puts Log.color :blue, path
+    path.glob("*").each do |subpath|
+      if nocolor
+        puts subpath
+      else
+        color = File.directory?(subpath) ? :blue : nil
+        puts " " << Log.color(color, subpath)
+      end
+    end
+  else
+    if nocolor
+      puts path
+    else
+      color = File.exists?(path) ? (File.directory?(path) ? :blue : nil) : :red
+      puts Log.color color, path
+    end
+
+  end
+end
+

data/lib/rbbt/hpc/slurm.rb

@@ -51,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "nodes" => nodes,
                        "time" => time,
                        "exclusive" => exclusive,
+                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,

data/lib/rbbt/tsv/excel.rb

@@ -177,7 +177,7 @@ module TSV
 
       sheet ||= "0"
       workbook = RubyXL::Parser.parse file
-      if sheet &&  sheet =~ /^\d+$/
+      if sheet && sheet =~ /^\d+$/
         sheet = workbook.worksheets.collect{|s| s.sheet_name }[sheet.to_i]
       end
       sheet_name = sheet
@@ -185,7 +185,9 @@ module TSV
 
       TmpFile.with_file :extension => Misc.sanitize_filename(sheet_name.to_s) do |filename|
 
-        sheet    = sheet ? workbook[sheet] : workbook.worksheets.first
+        sheet    = sheet_name ? workbook[sheet_name] : workbook.worksheets.first
+
+        raise "No sheet #{sheet_name} found" if sheet.nil?
 
         rows = []
 
@@ -217,21 +219,27 @@ module TSV
     end
 
     def self.write(tsv, file, options = {})
-      sheet = Misc.process_options options, :sheet
+      sheet, add_sheet = Misc.process_options options, :sheet, :add_sheet
 
       fields, rows = TSV._excel_data(tsv, options)
 
-      book = RubyXL::Workbook.new
-      sheet1 = book.worksheets.first
-      sheet1.sheet_name = sheet if sheet
+      if Open.exists?(file) && add_sheet
+        book = RubyXL::Parser.parse file
+        sheet1 = book.add_worksheet(sheet)
+      else
+        book = RubyXL::Workbook.new
+        sheet1 = book.worksheets.first
+        sheet1.sheet_name = sheet if sheet
+      end
 
       fields.each_with_index do |e,i|
         sheet1.add_cell(0, i, e)
-      end
+      end if fields
 
       rows.each_with_index do |cells,i|
+        i += 1 if fields
         cells.each_with_index do |e,j|
-          sheet1.add_cell(i+1, j, e)
+          sheet1.add_cell(i, j, e)
         end
       end
 

data/lib/rbbt/tsv/manipulate.rb

@@ -356,6 +356,16 @@ module TSV
     elems.sort_by{|k,v| v}.collect{|k,v| k}
   end
 
+  def subset(keys)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k]
+      end
+    end
+    new
+  end
+
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 

data/lib/rbbt/workflow/definition.rb

@@ -76,7 +76,7 @@ module Workflow
     extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
-      raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join

data/lib/rbbt/workflow/examples.rb

@@ -25,71 +25,6 @@ module Workflow
     end.compact
   end
 
-  def self.load_inputs(dir, input_names, input_types)
-    inputs = {}
-    if File.exists?(dir) && ! File.directory?(dir)
-      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-      tarfile = dir
-      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-      tmpdir = Rbbt.tmp.input_bundle[digest].find
-      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-      files = tmpdir.glob("*")
-      if files.length == 1 && File.directory?(files.first)
-        tmpdir = files.first
-      end
-      load_inputs(tmpdir, input_names, input_types)
-    else
-      dir = Path.setup(dir.dup)
-      input_names.each do |input|
-        file = dir[input].find
-        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not file.exists?
-        Log.debug "Trying #{ input }: #{file}"
-        next unless file and file.exists?
-
-
-        case input_types[input]
-        when :file
-          Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.yaml/
-            inputs[input.to_sym]  = YAML.load(Open.read(file))
-          else
-            if File.symlink?(file)
-              link_target = File.expand_path(File.readlink(file), File.dirname(file))
-              inputs[input.to_sym]  = link_target
-            else
-              inputs[input.to_sym]  = Open.realpath(file)
-            end
-          end
-        when :text
-          Log.debug "Reading #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file)
-        when :array
-          Log.debug "Reading array #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file).split("\n")
-        when :tsv
-          Log.debug "Opening tsv #{ input } from #{file}"
-          inputs[input.to_sym]  = TSV.open(file)
-        when :boolean
-          inputs[input.to_sym]  = (file.read.strip == 'true')
-        else
-          Log.debug "Loading #{ input } from #{file}"
-          inputs[input.to_sym]  = file.read.strip
-        end
-
-      end
-      inputs = IndiferentHash.setup(inputs)
-
-      dir.glob("*#*").each do |od|
-        name = File.basename(od)
-        value = Open.read(od)
-        Log.debug "Loading override dependency #{ name } as #{value}"
-        inputs[name] = value.chomp
-      end
-
-      inputs
-    end
-  end
-
   def example_inputs(task_name, example)
     inputs = {}
     IndiferentHash.setup(inputs)

data/lib/rbbt/workflow/step/accessor.rb

@@ -86,76 +86,6 @@ class Step
     end
   end
 
-  def self.save_inputs(inputs, input_types, dir)
-    inputs.each do |name,value|
-      type = input_types[name]
-      type = type.to_s if type
-      path = File.join(dir, name.to_s)
-
-      Log.debug "Saving job input #{name} (#{type}) into #{path}"
-      case
-      when Step === value
-        Open.ln_s(value.path, path)
-      when type.to_s == "file"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        else
-          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
-          value = "#{value}" if Path === value
-          Open.write(path + '.yaml', value.to_yaml)
-        end
-      when Array === value
-        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
-      when IO === value
-        if value.filename && String === value.filename && File.exists?(value.filename)
-          Open.ln_s(value.filename, path)
-        else
-          Open.write(path, value)
-        end
-      else
-        Open.write(path, value.to_s)
-      end
-    end.any?
-  end
-
-  def self.save_job_inputs(job, dir, options = nil)
-    options = IndiferentHash.setup options.dup if options
-
-    task_name = Symbol === job.overriden ? job.overriden : job.task_name
-    workflow = job.workflow
-    workflow = Kernel.const_get workflow if String === workflow
-    if workflow
-      task_info = IndiferentHash.setup(workflow.task_info(task_name))
-      input_types = IndiferentHash.setup(task_info[:input_types])
-      task_inputs = IndiferentHash.setup(task_info[:inputs])
-      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
-    else
-      task_info = IndiferentHash.setup({})
-      input_types = IndiferentHash.setup({})
-      task_inputs = IndiferentHash.setup({})
-      input_defaults = IndiferentHash.setup({})
-    end
-
-    inputs = IndiferentHash.setup({})
-    real_inputs = job.real_inputs || job.info[:real_inputs]
-    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
-      next unless task_inputs.include? name.to_sym
-      next unless real_inputs.include? name.to_sym
-      next if options && ! options.include?(name)
-      next if value.nil?
-      next if input_defaults[name] == value
-      inputs[name] = value
-    end
-
-    if options && options.include?('override_dependencies')
-      inputs.merge!(:override_dependencies => open[:override_dependencies])
-      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
-    end
-    save_inputs(inputs, input_types, dir)
-
-    inputs.keys
-  end
 
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')

data/lib/rbbt/workflow/step/dependencies.rb

@@ -254,7 +254,12 @@ class Step
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
 
-      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
+      if cpus.nil? 
+        keys = ['bootstrap'] + list.collect{|d| [d.task_name, d.task_signature] }.flatten.uniq
+        cpus = config('dep_cpus', *keys, :default => [5, list.length / 2].min)
+      elsif Symbol === cpus
+        cpus = config('dep_cpus', cpus, :default => [5, list.length / 2].min)
+      end
 
       respawn = rest && rest.include?(:respawn)
       respawn = false if rest && rest.include?(:norespawn)
@@ -369,7 +374,8 @@ class Step
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next unless step.dependencies.include?(step_dep)
-        next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
+        next if step_dep.done? or step_dep.running? or 
+          (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -0,0 +1,162 @@
+module Workflow
+  def self.load_inputs(dir, input_names, input_types)
+    inputs = {}
+    if File.exists?(dir) && ! File.directory?(dir)
+      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+      tarfile = dir
+      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+      tmpdir = Rbbt.tmp.input_bundle[digest].find
+      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+      files = tmpdir.glob("*")
+      if files.length == 1 && File.directory?(files.first)
+        tmpdir = files.first
+      end
+      load_inputs(tmpdir, input_names, input_types)
+    else
+      dir = Path.setup(dir.dup)
+      input_names.each do |input|
+        file = dir[input].find
+        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+        Log.debug "Trying #{ input }: #{file}"
+        next unless file and (File.symlink?(file) || file.exists?)
+
+        type = input_types[input]
+
+        type = :io if file.split(".").last == 'as_io'
+
+        case type
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :file, :binary
+          Log.debug "Pointing #{ input } to #{file}"
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
+          else
+            if File.symlink?(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
+            else
+              inputs[input.to_sym]  = Open.realpath(file)
+            end
+          end
+        when :text
+          Log.debug "Reading #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file)
+        when :array
+          Log.debug "Reading array #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file).split("\n")
+        when :tsv
+          Log.debug "Opening tsv #{ input } from #{file}"
+          inputs[input.to_sym]  = TSV.open(file)
+        when :boolean
+          inputs[input.to_sym]  = (file.read.strip == 'true')
+        else
+          Log.debug "Loading #{ input } from #{file}"
+          inputs[input.to_sym]  = file.read.strip
+        end
+
+      end
+      inputs = IndiferentHash.setup(inputs)
+
+      dir.glob("*#*").each do |od|
+        name = File.basename(od)
+        value = Open.read(od)
+        Log.debug "Loading override dependency #{ name } as #{value}"
+        inputs[name] = value.chomp
+      end
+
+      inputs
+    end
+  end
+
+  def task_inputs_from_directory(task_name, directory)
+    task_info = self.task_info(task_name)
+    Workflow.load_inputs(directory, task_info[:inputs], task_info[:input_types])
+  end
+
+  def job_for_directory_inputs(task_name, directory, jobname = nil)
+    inputs = task_inputs_from_directory(task_name, directory)
+    job(task_name, jobname, inputs)
+  end
+
+end
+
+class Step
+  def self.save_inputs(inputs, input_types, input_options, dir)
+    inputs.each do |name,value|
+      type = input_types[name]
+      type = type.to_s if type
+      path = File.join(dir, name.to_s)
+
+      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+      Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+      case
+      when IO === value
+        Open.write(path, value.to_s)
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        else
+          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
+          value = "#{value}" if Path === value
+          Open.write(path + '.yaml', value.to_yaml)
+        end
+      when Array === value
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+      when IO === value
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
+      else
+        Open.write(path, value.to_s)
+      end
+    end.any?
+  end
+
+  def self.save_job_inputs(job, dir, options = nil)
+    options = IndiferentHash.setup options.dup if options
+
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
+    workflow = job.workflow
+    workflow = Kernel.const_get workflow if String === workflow
+    if workflow
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      input_options = IndiferentHash.setup(task_info[:input_options])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
+    else
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      task_options = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
+    end
+
+    inputs = IndiferentHash.setup({})
+    real_inputs = job.real_inputs || job.info[:real_inputs]
+    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
+      next unless task_inputs.include? name.to_sym
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
+      next if value.nil?
+      next if input_defaults[name] == value
+      inputs[name] = value
+    end
+
+    if options && options.include?('override_dependencies')
+      inputs.merge!(:override_dependencies => open[:override_dependencies])
+      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
+    end
+
+    save_inputs(inputs, input_types, input_options, dir)
+
+    inputs.keys
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -6,6 +6,7 @@ require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
+require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -321,7 +322,7 @@ class Step
   def load
     res = begin
             @result = nil if IO === @result && @result.closed?
-            if @result && @path != @result
+            if @result && @path != @result &&  ! StreamArray === @result
               res = @result
             else
               join if not done?

data/lib/rbbt/workflow/task.rb

@@ -90,11 +90,11 @@ module Task
 
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name]
+      next if seen.include? [wf, task.name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name]
+      seen << [wf, task.name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/lib/rbbt/workflow.rb

@@ -45,14 +45,10 @@ module Workflow
     load_remote_tasks(Rbbt.root.etc.remote_tasks.find) if Rbbt.root.etc.remote_tasks.exists?
   end
 
-  def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
-  end
 
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
+    eval "Object::#{wf_name.split("+").first} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
   end
 
   def self.load_workflow_libdir(filename)
@@ -134,9 +130,10 @@ module Workflow
   end
 
   def self.require_local_workflow(wf_name)
+
     filename = local_workflow_filename(wf_name)
 
-    if filename and File.exist? filename
+    if filename and File.exist?(filename)
       load_workflow_file filename
     else
       return false
@@ -194,14 +191,23 @@ module Workflow
     end
 
     Log.high{"Loading workflow #{wf_name}"}
-    require_local_workflow(wf_name) or 
-    (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
-    workflow = begin
-                 Misc.string2const Misc.camel_case(wf_name)
-               rescue
-                 Workflow.workflows.last || true
-               end
-    workflow.load_documentation
+
+    first = nil
+    wf_name.split("+").each do |wf_name|
+      require_local_workflow(wf_name) or 
+        (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
+
+      workflow = begin
+                   Misc.string2const Misc.camel_case(wf_name.split("+").first)
+                 rescue
+                   Workflow.workflows.last || true
+                 end
+      workflow.load_documentation
+
+      first ||= workflow
+    end
+    return first
+
     workflow
   end
 

data/share/rbbt_commands/tsv/read_excel

@@ -22,7 +22,7 @@ Use - to read from STDIN
 -h--help Print this help
 -s--sheet* Sheet to extract
 -skip--skip_rows* Initial rows to skip
-
+-o--original Dump the rows without parsing them into TSV
 EOF
 if options[:help]
   if defined? rbbt_usage
@@ -39,5 +39,5 @@ raise ParameterException, "No excel file given" if excelfile.nil?
 
 options[:zipped] ||= true if options[:merge]
 require 'rbbt/tsv/excel'
-puts TSV.excel(excelfile, options).to_s
+puts TSV.excel(excelfile, options.merge(:text => options[:original]))
 

data/share/rbbt_commands/workflow/task

@@ -104,7 +104,7 @@ def fix_options(workflow, task, job_options)
               elsif input_options[name] and input_options[name][:stream] and value == "-"
                 STDIN
               else
-                if Array === value
+                if Array === value || IO === value
                   value
                 else
                   array_separator = $array_separator
@@ -137,6 +137,8 @@ def fix_options(workflow, task, job_options)
                   TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 when (Misc.is_filename?(value) and String)
                   TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
+                when IO
+                  TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 else
                   TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 end
@@ -287,8 +289,8 @@ else
     puts
     puts $!.message
     puts
-    
-    exit 0
+
+    exit -1
   end
 end
 
@@ -572,7 +574,9 @@ when Step
     exit! 0
   else
     res.join
-    out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    Open.open(res.path, :mode => 'rb') do |io|
+      Misc.consume_stream(io, false, out)
+    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
   end
 else
   if Array === res

data/test/rbbt/tsv/test_excel.rb

@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/tsv/excel'
 
 class TestExcel < Test::Unit::TestCase
-  def _test_xls
+  def test_xls
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -19,7 +19,7 @@ row2    A    B    Id3
     end
   end
 
-  def _test_xlsx
+  def test_xlsx
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -36,7 +36,7 @@ row2    A    B    Id3
     end
   end
 
-  def _test_excel
+  def test_excel
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -63,7 +63,7 @@ row2    A    B    Id3
     end
   end
   
-  def _test_excel_sheets
+  def test_excel_sheets
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -133,5 +133,39 @@ row2    A    B    Id3
       end
     end
   end
+
+  def test_excel_multi_sheets
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv1 = TSV.open(filename, :sep => /\s+/)
+      tsv2 = tsv1.annotate(tsv1.dup)
+      tsv3 = tsv1.annotate(tsv1.dup)
+
+      tsv2["row2"] = [["AA"], ["BB"], ["Id4"]]
+      tsv3["row2"] = [["AAA"], ["BBB"], ["Id5"]]
+
+      TmpFile.with_file(nil, false, :extension => 'xlsx') do |excelfile|
+        tsv1.xlsx(excelfile, :sheet => "S1")
+        tsv2.xlsx(excelfile, :sheet => "S2", :add_sheet => true)
+        workbook = RubyXL::Parser.parse excelfile
+
+        assert_equal %w(S1 S2), workbook.worksheets.collect{|s| s.sheet_name} 
+
+        new = TSV.excel(excelfile, :sheet => "S1")
+        assert_equal %w(row1 row2), new.keys.sort
+        assert_equal %w(A), new["row2"]["ValueA"]
+
+        new = TSV.excel(excelfile, :sheet => "S2")
+        assert_equal %w(row1 row2), new.keys.sort
+        assert_equal %w(AA), new["row2"]["ValueA"]
+
+      end
+    end
+  end
 end
 

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -203,20 +203,21 @@ class TestWorkflowDependency < Test::Unit::TestCase
     size = 100000
     content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
-    Log.severity = 0
-    TmpFile.with_file(content) do |input_file|
-      begin
-        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
-        job.recursive_clean
-        job.run(:stream)
-        io = TSV.get_stream job
-        while line = io.gets
-          last_line = line.strip
+    Log.with_severity 0 do
+      TmpFile.with_file(content) do |input_file|
+        begin
+          job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+          job.recursive_clean
+          job.run(:stream)
+          io = TSV.get_stream job
+          while line = io.gets
+            last_line = line.strip
+          end
+          io.join if io.respond_to? :join
+        rescue Exception
+          job.abort
+          raise $!
         end
-        io.join if io.respond_to? :join
-      rescue Exception
-        job.abort
-        raise $!
       end
     end
     assert last_line.include? "Line #{size}"

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -0,0 +1,46 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/step/save_load_inputs'
+
+ENV["RBBT_DEBUG_JOB_HASH"] = true.to_s
+require 'rbbt/workflow'
+module TestSaveLoadWF
+  extend Workflow
+
+  task :number => :integer do
+    10
+  end
+
+  task :list => :array do
+    (0..10).to_a.collect{|e| e.to_s}
+  end
+
+  input :list, :array
+  input :number, :integer
+  task :reverse => :array do |list|
+    list.reverse
+  end
+
+  dep :list
+  dep :number
+  dep :reverse, :list => :list, :number => :number
+  task :prefix => :array do
+    step(:reverse).run.collect{|e| "A-#{e}" }
+  end
+end
+
+class TestSaveLoad < Test::Unit::TestCase
+  def test_save
+    Log.with_severity 0 do
+    job = TestSaveLoadWF.job(:prefix)
+    job.recursive_clean
+    job = TestSaveLoadWF.job(:prefix)
+    TmpFile.with_file do |directory|
+      Step.save_job_inputs(job.step(:reverse), directory)
+      job.produce
+      newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
+      assert_equal job.rec_dependencies.last.path, newjob.path
+    end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.23
+  version: 5.32.27
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-28 00:00:00.000000000 Z
+date: 2021-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -172,6 +172,7 @@ executables:
 - rbbt_Rutil.rb
 - rbbt
 - rbbt_dangling_locks.rb
+- rbbt_find.rb
 extensions: []
 extra_rdoc_files:
 - LICENSE
@@ -183,6 +184,7 @@ files:
 - bin/rbbt_Rutil.rb
 - bin/rbbt_dangling_locks.rb
 - bin/rbbt_exec.rb
+- bin/rbbt_find.rb
 - bin/rbbt_query.rb
 - etc/app.d/base.rb
 - etc/app.d/entities.rb
@@ -343,6 +345,7 @@ files:
 - lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/save_load_inputs.rb
 - lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
@@ -548,6 +551,7 @@ files:
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/test_schedule.rb
@@ -590,6 +594,7 @@ test_files:
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb