rbbt-util 5.32.26 → 5.32.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 072aa8b010c3f395051fd4e29f6ac22647944eec529bdf05be77ac6ad5260a37
-  data.tar.gz: d68843b2fe56019dd63165a34c97ec23d9e60933e4493bdb61f29985548bca69
+  metadata.gz: ffa357551766aa6ae6da35c586949c0ff05814171304ca5a874689418c7c48e2
+  data.tar.gz: 5e1e419fcf846fc8ead12a77e6713f198a74d516c0262c34b4d86380a58af092
 SHA512:
-  metadata.gz: 9d572a394b5736d8780021bc8ec65ba4381dc8cb84f111704854ce77ceca8da0ca72626e0144a04ab5c9b552a5566f4bdd553cad2a70050216fae231179beb95
-  data.tar.gz: ce51f8a4f0f5b4028148deb7fd3e57a4fbf5bb531f366378a2bae31354b799a66c6ba2768d9ae25578b3a0b09ee4e29f817f6ab768166bed39e4245b35a4a7e4
+  metadata.gz: a6afd86e7698e5b7e4588412ed678497c825242616bc75d2fa2ab835bc325a8a58826da164dea16cd5b4bbf81697ac5201bb347af220bab5ce238f8e0ce4edcb
+  data.tar.gz: 3e86fe7c2d84cc7a04cdd348f03a6323b5467723182057ec028b5d611b3dd200fce41949f1502ca59b0a295b5664f1d6ba259d27a196039f873261e9faf7a1a6

data/lib/rbbt/hpc/slurm.rb

@@ -51,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "nodes" => nodes,
                        "time" => time,
                        "exclusive" => exclusive,
+                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,

data/lib/rbbt/workflow/definition.rb

@@ -76,7 +76,7 @@ module Workflow
     extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
-      raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join

data/lib/rbbt/workflow/examples.rb

@@ -25,71 +25,6 @@ module Workflow
     end.compact
   end
 
-  def self.load_inputs(dir, input_names, input_types)
-    inputs = {}
-    if File.exists?(dir) && ! File.directory?(dir)
-      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-      tarfile = dir
-      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-      tmpdir = Rbbt.tmp.input_bundle[digest].find
-      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-      files = tmpdir.glob("*")
-      if files.length == 1 && File.directory?(files.first)
-        tmpdir = files.first
-      end
-      load_inputs(tmpdir, input_names, input_types)
-    else
-      dir = Path.setup(dir.dup)
-      input_names.each do |input|
-        file = dir[input].find
-        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not file.exists?
-        Log.debug "Trying #{ input }: #{file}"
-        next unless file and file.exists?
-
-
-        case input_types[input]
-        when :file, :binary
-          Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.yaml/
-            inputs[input.to_sym]  = YAML.load(Open.read(file))
-          else
-            if File.symlink?(file)
-              link_target = File.expand_path(File.readlink(file), File.dirname(file))
-              inputs[input.to_sym]  = link_target
-            else
-              inputs[input.to_sym]  = Open.realpath(file)
-            end
-          end
-        when :text
-          Log.debug "Reading #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file)
-        when :array
-          Log.debug "Reading array #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file).split("\n")
-        when :tsv
-          Log.debug "Opening tsv #{ input } from #{file}"
-          inputs[input.to_sym]  = TSV.open(file)
-        when :boolean
-          inputs[input.to_sym]  = (file.read.strip == 'true')
-        else
-          Log.debug "Loading #{ input } from #{file}"
-          inputs[input.to_sym]  = file.read.strip
-        end
-
-      end
-      inputs = IndiferentHash.setup(inputs)
-
-      dir.glob("*#*").each do |od|
-        name = File.basename(od)
-        value = Open.read(od)
-        Log.debug "Loading override dependency #{ name } as #{value}"
-        inputs[name] = value.chomp
-      end
-
-      inputs
-    end
-  end
-
   def example_inputs(task_name, example)
     inputs = {}
     IndiferentHash.setup(inputs)

data/lib/rbbt/workflow/step/accessor.rb

@@ -86,76 +86,6 @@ class Step
     end
   end
 
-  def self.save_inputs(inputs, input_types, dir)
-    inputs.each do |name,value|
-      type = input_types[name]
-      type = type.to_s if type
-      path = File.join(dir, name.to_s)
-
-      Log.debug "Saving job input #{name} (#{type}) into #{path}"
-      case
-      when Step === value
-        Open.ln_s(value.path, path)
-      when type.to_s == "file"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        else
-          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
-          value = "#{value}" if Path === value
-          Open.write(path + '.yaml', value.to_yaml)
-        end
-      when Array === value
-        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
-      when IO === value
-        if value.filename && String === value.filename && File.exists?(value.filename)
-          Open.ln_s(value.filename, path)
-        else
-          Open.write(path, value)
-        end
-      else
-        Open.write(path, value.to_s)
-      end
-    end.any?
-  end
-
-  def self.save_job_inputs(job, dir, options = nil)
-    options = IndiferentHash.setup options.dup if options
-
-    task_name = Symbol === job.overriden ? job.overriden : job.task_name
-    workflow = job.workflow
-    workflow = Kernel.const_get workflow if String === workflow
-    if workflow
-      task_info = IndiferentHash.setup(workflow.task_info(task_name))
-      input_types = IndiferentHash.setup(task_info[:input_types])
-      task_inputs = IndiferentHash.setup(task_info[:inputs])
-      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
-    else
-      task_info = IndiferentHash.setup({})
-      input_types = IndiferentHash.setup({})
-      task_inputs = IndiferentHash.setup({})
-      input_defaults = IndiferentHash.setup({})
-    end
-
-    inputs = IndiferentHash.setup({})
-    real_inputs = job.real_inputs || job.info[:real_inputs]
-    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
-      next unless task_inputs.include? name.to_sym
-      next unless real_inputs.include? name.to_sym
-      next if options && ! options.include?(name)
-      next if value.nil?
-      next if input_defaults[name] == value
-      inputs[name] = value
-    end
-
-    if options && options.include?('override_dependencies')
-      inputs.merge!(:override_dependencies => open[:override_dependencies])
-      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
-    end
-    save_inputs(inputs, input_types, dir)
-
-    inputs.keys
-  end
 
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')

data/lib/rbbt/workflow/step/dependencies.rb

@@ -254,7 +254,12 @@ class Step
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
 
-      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
+      if cpus.nil? 
+        keys = ['bootstrap'] + list.collect{|d| [d.task_name, d.task_signature] }.flatten.uniq
+        cpus = config('dep_cpus', *keys, :default => [5, list.length / 2].min)
+      elsif Symbol === cpus
+        cpus = config('dep_cpus', cpus, :default => [5, list.length / 2].min)
+      end
 
       respawn = rest && rest.include?(:respawn)
       respawn = false if rest && rest.include?(:norespawn)
@@ -369,7 +374,8 @@ class Step
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next unless step.dependencies.include?(step_dep)
-        next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
+        next if step_dep.done? or step_dep.running? or 
+          (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -0,0 +1,162 @@
+module Workflow
+  def self.load_inputs(dir, input_names, input_types)
+    inputs = {}
+    if File.exists?(dir) && ! File.directory?(dir)
+      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+      tarfile = dir
+      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+      tmpdir = Rbbt.tmp.input_bundle[digest].find
+      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+      files = tmpdir.glob("*")
+      if files.length == 1 && File.directory?(files.first)
+        tmpdir = files.first
+      end
+      load_inputs(tmpdir, input_names, input_types)
+    else
+      dir = Path.setup(dir.dup)
+      input_names.each do |input|
+        file = dir[input].find
+        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+        Log.debug "Trying #{ input }: #{file}"
+        next unless file and (File.symlink?(file) || file.exists?)
+
+        type = input_types[input]
+
+        type = :io if file.split(".").last == 'as_io'
+
+        case type
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :file, :binary
+          Log.debug "Pointing #{ input } to #{file}"
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
+          else
+            if File.symlink?(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
+            else
+              inputs[input.to_sym]  = Open.realpath(file)
+            end
+          end
+        when :text
+          Log.debug "Reading #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file)
+        when :array
+          Log.debug "Reading array #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file).split("\n")
+        when :tsv
+          Log.debug "Opening tsv #{ input } from #{file}"
+          inputs[input.to_sym]  = TSV.open(file)
+        when :boolean
+          inputs[input.to_sym]  = (file.read.strip == 'true')
+        else
+          Log.debug "Loading #{ input } from #{file}"
+          inputs[input.to_sym]  = file.read.strip
+        end
+
+      end
+      inputs = IndiferentHash.setup(inputs)
+
+      dir.glob("*#*").each do |od|
+        name = File.basename(od)
+        value = Open.read(od)
+        Log.debug "Loading override dependency #{ name } as #{value}"
+        inputs[name] = value.chomp
+      end
+
+      inputs
+    end
+  end
+
+  def task_inputs_from_directory(task_name, directory)
+    task_info = self.task_info(task_name)
+    Workflow.load_inputs(directory, task_info[:inputs], task_info[:input_types])
+  end
+
+  def job_for_directory_inputs(task_name, directory, jobname = nil)
+    inputs = task_inputs_from_directory(task_name, directory)
+    job(task_name, jobname, inputs)
+  end
+
+end
+
+class Step
+  def self.save_inputs(inputs, input_types, input_options, dir)
+    inputs.each do |name,value|
+      type = input_types[name]
+      type = type.to_s if type
+      path = File.join(dir, name.to_s)
+
+      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+      Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+      case
+      when IO === value
+        Open.write(path, value.to_s)
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        else
+          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
+          value = "#{value}" if Path === value
+          Open.write(path + '.yaml', value.to_yaml)
+        end
+      when Array === value
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+      when IO === value
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
+      else
+        Open.write(path, value.to_s)
+      end
+    end.any?
+  end
+
+  def self.save_job_inputs(job, dir, options = nil)
+    options = IndiferentHash.setup options.dup if options
+
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
+    workflow = job.workflow
+    workflow = Kernel.const_get workflow if String === workflow
+    if workflow
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      input_options = IndiferentHash.setup(task_info[:input_options])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
+    else
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      task_options = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
+    end
+
+    inputs = IndiferentHash.setup({})
+    real_inputs = job.real_inputs || job.info[:real_inputs]
+    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
+      next unless task_inputs.include? name.to_sym
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
+      next if value.nil?
+      next if input_defaults[name] == value
+      inputs[name] = value
+    end
+
+    if options && options.include?('override_dependencies')
+      inputs.merge!(:override_dependencies => open[:override_dependencies])
+      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
+    end
+
+    save_inputs(inputs, input_types, input_options, dir)
+
+    inputs.keys
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -6,6 +6,7 @@ require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
+require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -321,7 +322,7 @@ class Step
   def load
     res = begin
             @result = nil if IO === @result && @result.closed?
-            if @result && @path != @result
+            if @result && @path != @result &&  ! StreamArray === @result
               res = @result
             else
               join if not done?

data/lib/rbbt/workflow/task.rb

@@ -90,11 +90,11 @@ module Task
 
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name]
+      next if seen.include? [wf, task.name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name]
+      seen << [wf, task.name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/share/rbbt_commands/tsv/read_excel

@@ -22,7 +22,7 @@ Use - to read from STDIN
 -h--help Print this help
 -s--sheet* Sheet to extract
 -skip--skip_rows* Initial rows to skip
-
+-o--original Dump the rows without parsing them into TSV
 EOF
 if options[:help]
   if defined? rbbt_usage
@@ -39,5 +39,5 @@ raise ParameterException, "No excel file given" if excelfile.nil?
 
 options[:zipped] ||= true if options[:merge]
 require 'rbbt/tsv/excel'
-puts TSV.excel(excelfile, options).to_s
+puts TSV.excel(excelfile, options.merge(:text => options[:original]))
 

data/share/rbbt_commands/workflow/task

@@ -104,7 +104,7 @@ def fix_options(workflow, task, job_options)
               elsif input_options[name] and input_options[name][:stream] and value == "-"
                 STDIN
               else
-                if Array === value
+                if Array === value || IO === value
                   value
                 else
                   array_separator = $array_separator
@@ -137,6 +137,8 @@ def fix_options(workflow, task, job_options)
                   TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 when (Misc.is_filename?(value) and String)
                   TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
+                when IO
+                  TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 else
                   TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -203,20 +203,21 @@ class TestWorkflowDependency < Test::Unit::TestCase
     size = 100000
     content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
-    Log.severity = 0
-    TmpFile.with_file(content) do |input_file|
-      begin
-        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
-        job.recursive_clean
-        job.run(:stream)
-        io = TSV.get_stream job
-        while line = io.gets
-          last_line = line.strip
+    Log.with_severity 0 do
+      TmpFile.with_file(content) do |input_file|
+        begin
+          job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+          job.recursive_clean
+          job.run(:stream)
+          io = TSV.get_stream job
+          while line = io.gets
+            last_line = line.strip
+          end
+          io.join if io.respond_to? :join
+        rescue Exception
+          job.abort
+          raise $!
         end
-        io.join if io.respond_to? :join
-      rescue Exception
-        job.abort
-        raise $!
       end
     end
     assert last_line.include? "Line #{size}"

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -0,0 +1,46 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/step/save_load_inputs'
+
+ENV["RBBT_DEBUG_JOB_HASH"] = true.to_s
+require 'rbbt/workflow'
+module TestSaveLoadWF
+  extend Workflow
+
+  task :number => :integer do
+    10
+  end
+
+  task :list => :array do
+    (0..10).to_a.collect{|e| e.to_s}
+  end
+
+  input :list, :array
+  input :number, :integer
+  task :reverse => :array do |list|
+    list.reverse
+  end
+
+  dep :list
+  dep :number
+  dep :reverse, :list => :list, :number => :number
+  task :prefix => :array do
+    step(:reverse).run.collect{|e| "A-#{e}" }
+  end
+end
+
+class TestSaveLoad < Test::Unit::TestCase
+  def test_save
+    Log.with_severity 0 do
+    job = TestSaveLoadWF.job(:prefix)
+    job.recursive_clean
+    job = TestSaveLoadWF.job(:prefix)
+    TmpFile.with_file do |directory|
+      Step.save_job_inputs(job.step(:reverse), directory)
+      job.produce
+      newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
+      assert_equal job.rec_dependencies.last.path, newjob.path
+    end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.26
+  version: 5.32.27
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-10 00:00:00.000000000 Z
+date: 2021-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -345,6 +345,7 @@ files:
 - lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/save_load_inputs.rb
 - lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
@@ -550,6 +551,7 @@ files:
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/test_schedule.rb
@@ -592,6 +594,7 @@ test_files:
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb