rbbt-util 5.33.8 → 5.33.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaa014ffdad4e098212cc3f7cefc4d04ac62a7f7fcbf17bbbab9415c6253d9ef
-  data.tar.gz: b8a2a233686ec2f229ed79bb19d87d8eb79ad945142305f7b61dcdb11f5f7912
+  metadata.gz: 5c1ff60078b8ea97b0d68c7ec8beb1831889cd0e4ef841c781073dad938924be
+  data.tar.gz: 75dd8c9528cc2015c83b3d106027a5bdc29ba5d9eca53b8024ab19eb824c5273
 SHA512:
-  metadata.gz: 564242bef1dd92c2f53a5fe6c967624de26d7ae7eeb4da1ef2a1f682db9cf6d4b0bf0f8e76b3a600afe6af7bb443df28a8fe50e570c3c56265319eb607d6c1fc
-  data.tar.gz: ac7c01fa7e8fccdff8dfc861b82f28889a68f77886c198ff55b09558b36857ec9c29a8bae979bc7d088f311ad81515676bc7a4cc30097fd6c4b7586e6838c0e3
+  metadata.gz: 60379d7fd62c52f115a198bab61ef24ba2dc4be5c0fba9d4974637afaea9481b99530d0cadbae07b03db92fec75dee1ea1bb23d3fa96906d6cc918794ab0973b
+  data.tar.gz: 7c7ffd9b0847083722e6976f98eb700fef719f7fba7b36914e690538a93de047c7c46d42c0715ac430edc9c392ad5b0f1b0e97de65f812ba2eed3835263dbbe9

data/lib/rbbt/hpc/orchestrate.rb

@@ -30,6 +30,7 @@ module HPC
       options.delete "printpath"
       options.delete "detach"
       options.delete "jobname"
+      options.delete "load_inputs"
 
       Log.high "Prepare for exec"
       prepare_for_execution(job)

data/lib/rbbt/hpc/slurm.rb

@@ -94,8 +94,8 @@ export BATCH_SYSTEM=SLURM
 
       return if Open.exists?(fexit)
 
-      STDERR.puts Log.color(:magenta, "Issuing SLURM file: #{fcmd}")
-      STDERR.puts Open.read(fcmd)
+      Log.info "Issuing SLURM file: #{fcmd}"
+      Log.debug Open.read(fcmd)
 
       if File.exists?(fjob)
         job = Open.read(fjob).to_i

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -1,4 +1,5 @@
 module Workflow
+
   def self.load_inputs(dir, input_names, input_types)
     inputs = {}
     if File.exists?(dir) && ! File.directory?(dir)
@@ -24,17 +25,31 @@ module Workflow
 
         type = :io if file.split(".").last == 'as_io'
 
+        type = :step if file.split(".").last == 'as_step'
+
+        type = :step_file if file.split(".").last == 'as_step_file'
+
         type = :path if file.split(".").last == 'as_path'
 
+        type = :filename if file.split(".").last == 'as_filename'
+
         type = :nofile if file.split(".").last == 'nofile'
 
         case type
         when :nofile
           inputs[input.to_sym]  = Open.realpath(file)
         when :path
-          inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+          inputs[input.to_sym]  = Open.read(file).strip
         when :io
           inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :step
+          inputs[input.to_sym] = Workflow.load_step(Open.read(file).strip)
+        when :step_file
+          path = Open.read(file).strip
+          path.extend Path
+          step_path = path.match(/(.*)\.files/)[1]
+          path.resource = Step.new step_path
+          inputs[input.to_sym] = path
         when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
@@ -87,52 +102,124 @@ module Workflow
     job(task_name, jobname, inputs)
   end
 
+  #def self.load_inputs_old(dir, input_names, input_types)
+  #  inputs = {}
+  #  if File.exists?(dir) && ! File.directory?(dir)
+  #    Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+  #    tarfile = dir
+  #    digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+  #    tmpdir = Rbbt.tmp.input_bundle[digest].find
+  #    Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+  #    files = tmpdir.glob("*")
+  #    if files.length == 1 && File.directory?(files.first)
+  #      tmpdir = files.first
+  #    end
+  #    load_inputs(tmpdir, input_names, input_types)
+  #  else
+  #    dir = Path.setup(dir.dup)
+  #    input_names.each do |input|
+  #      file = dir[input].find
+  #      file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+  #      Log.debug "Trying #{ input }: #{file}"
+  #      next unless file and (File.symlink?(file) || file.exists?)
+
+  #      type = input_types[input]
+
+  #      type = :io if file.split(".").last == 'as_io'
+
+  #      type = :path if file.split(".").last == 'as_path'
+
+  #      type = :filename if file.split(".").last == 'as_filename'
+
+  #      type = :nofile if file.split(".").last == 'nofile'
+
+  #      case type
+  #      when :nofile
+  #        inputs[input.to_sym]  = Open.realpath(file)
+  #      when :path
+  #        inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+  #      when :io
+  #        inputs[input.to_sym] = Open.open(Open.realpath(file))
+  #      when :file, :binary
+  #        Log.debug "Pointing #{ input } to #{file}"
+  #        if file =~ /\.yaml/
+  #          inputs[input.to_sym]  = YAML.load(Open.read(file))
+  #        else
+  #          if File.symlink?(file)
+  #            link_target = File.expand_path(File.readlink(file), File.dirname(file))
+  #            inputs[input.to_sym]  = link_target
+  #          else
+  #            inputs[input.to_sym]  = Open.realpath(file)
+  #          end
+  #        end
+  #      when :text
+  #        Log.debug "Reading #{ input } from #{file}"
+  #        inputs[input.to_sym]  = Open.read(file)
+  #      when :array
+  #        Log.debug "Reading array #{ input } from #{file}"
+  #        inputs[input.to_sym]  = Open.read(file).split("\n")
+  #      when :tsv
+  #        Log.debug "Opening tsv #{ input } from #{file}"
+  #        inputs[input.to_sym]  = TSV.open(file)
+  #      when :boolean
+  #        inputs[input.to_sym]  = (file.read.strip == 'true')
+  #      else
+  #        Log.debug "Loading #{ input } from #{file}"
+  #        inputs[input.to_sym]  = file.read.strip
+  #      end
+
+  #    end
+  #    inputs = IndiferentHash.setup(inputs)
+
+  #    dir.glob("*#*").each do |od|
+  #      name = File.basename(od)
+  #      value = Open.read(od)
+  #      Log.debug "Loading override dependency #{ name } as #{value}"
+  #      inputs[name] = value.chomp
+  #    end
+
+  #    inputs
+  #  end
+  #end
 end
 
 class Step
+  def self.save_input(name, value, type, dir)
+    path = File.join(dir, name.to_s)
+
+
+    case value
+    when Path
+      if Step === value.resource
+        path = path + '.as_step_file'
+      else
+        path = path + '.as_path'
+      end
+    when String
+      if Misc.is_filename?(value, false)
+        value = value.dup
+        value.extend Path
+        return save_input(name, value, type, dir)
+      end
+    when IO
+      path = path + '.as_io'
+    when Step
+      value = value.path
+      path = path + '.as_step'
+    when Array
+      value = value * "\n"
+    end
+
+    Log.debug "Saving job input #{name} (#{type}) into #{path}"
+    Open.write(path, value.to_s)
+  end
+
   def self.save_inputs(inputs, input_types, input_options, dir)
     inputs.each do |name,value|
       type = input_types[name]
       type = type.to_s if type
-      path = File.join(dir, name.to_s)
-
-      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
-      Log.debug "Saving job input #{name} (#{type}) into #{path}"
-
-      case
-      when IO === value
-        Open.write(path, value.to_s)
-      when Step === value
-        Open.ln_s(value.path, path)
-      when type.to_s == "binary"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        elsif String === value && Misc.is_filename?(value, false)
-          Open.write(path + '.as_path' , value)
-        else
-          Open.write(path, value, :mode => 'wb')
-        end
-      when type.to_s == "file"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        else
-          value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
-          value = "#{value}" if Path === value
-          Open.write(path + '.yaml', value.to_yaml)
-        end
-      when Array === value
-        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
-      when IO === value
-        if value.filename && String === value.filename && File.exists?(value.filename)
-          Open.ln_s(value.filename, path)
-        else
-          Open.write(path, value)
-        end
-      else
-        Open.write(path, value.to_s)
-      end
+
+      save_input(name, value, type, dir)
     end.any?
   end
 
@@ -176,4 +263,60 @@ class Step
 
     inputs.keys
   end
+
+  #def self.save_inputs_old(inputs, input_types, input_options, dir)
+  #  inputs.each do |name,value|
+  #    type = input_types[name]
+  #    type = type.to_s if type
+  #    path = File.join(dir, name.to_s)
+
+  #    if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+  #      path = path + '.as_io' 
+  #    elsif Misc.is_filename?(value, false)
+  #      path = path + '.as_filename'
+  #    end
+
+  #    Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+  #    case
+  #    when IO === value
+  #      Open.write(path, value.to_s)
+  #    when Step === value
+  #      Open.ln_s(value.path, path)
+  #    when type.to_s == "binary"
+  #      if String === value && File.exists?(value)
+  #        value = File.expand_path(value)
+  #        Open.ln_s(value, path)
+  #      elsif String === value && Misc.is_filename?(value, false)
+  #        Open.write(path + '.as_path' , value)
+  #      else
+  #        Open.write(path, value, :mode => 'wb')
+  #      end
+  #    when TSV === value
+  #      Open.write(path, value.to_s)
+  #    when Array === value
+  #      Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+  #    when %w(file tsv array).include?(type.to_s)
+  #      if String === value && File.exists?(value)
+  #        value = File.expand_path(value)
+  #        Open.ln_s(value, path)
+  #      elsif String === value && Misc.is_filename?(value, false)
+  #        Open.write(path + '.as_path' , value)
+  #      else
+  #        value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
+  #        value = "#{value}" if Path === value
+  #        Open.write(path + '.yaml', value.to_yaml)
+  #      end
+  #    when IO === value
+  #      if value.filename && String === value.filename && File.exists?(value.filename)
+  #        Open.ln_s(value.filename, path)
+  #      else
+  #        Open.write(path, value)
+  #      end
+  #    else
+  #      Open.write(path, value.to_s)
+  #    end
+  #  end.any?
+  #end
+
 end

data/lib/rbbt/workflow/step.rb

@@ -376,9 +376,9 @@ class Step
                        }
                        raise "Dependency step not found: #{ name }" if deps.empty?
                        if (deps & self.dependencies).any?
-                         (deps & self.dependencies).first
+                         (deps & self.dependencies).last
                        else
-                         deps.first
+                         deps.last
                        end
                      end
   end

data/share/rbbt_commands/hpc/list

@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/lsf/list

@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/slurm/list

@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    rescue
      nodes = []
    end
- elsif job_nodes[id]
+ elsif job_nodes && job_nodes[id]
    nodes = job_nodes[id].reject{|n| n.include? "("}
  else
    nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          has_bar = true
        end
      end
-     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
+     step_status = step.status
+     step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
+     step_status = Log.color :green, step_status if step_status.to_s == 'done'
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/workflow/task

@@ -76,6 +76,16 @@ def fix_options(workflow, task, job_options)
     type = input_types[name]
     type = type.to_sym if type
 
+    if Step === value
+        job_options_cleaned[name] = value
+        next
+    end
+
+    if Path === value && Step === value.resource
+        job_options_cleaned[name] = value
+        next
+    end
+
     value = case type
             when nil
                 value
@@ -148,6 +158,7 @@ def fix_options(workflow, task, job_options)
             else
               value
             end
+
     job_options_cleaned[name] = value
   end
 

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -26,11 +26,26 @@ module TestSaveLoadWF
   task :prefix => :array do
     step(:reverse).run.collect{|e| "A-#{e}" }
   end
+
+  input :integer, :integer
+  input :float, :float
+  input :file, :file
+  input :file_no_file, :file, "", nil, :nofile => true
+  input :string, :string
+  input :select, :select
+  input :array, :array
+  input :array_no_file, :array, "", nil, :nofile => true
+  input :tsv, :tsv, "", nil, :nofile => true
+  input :tsv_no_file, :tsv, "", nil, :nofile => true
+  input :binary, :binary, "", nil, :nofile => true
+  input :binary_no_file, :tsv, "", nil, :nofile => true
+  task :save_test => :string do
+    "DONE"
+  end
 end
 
 class TestSaveLoad < Test::Unit::TestCase
   def test_save
-    Log.with_severity 0 do
     job = TestSaveLoadWF.job(:prefix)
     job.recursive_clean
     job = TestSaveLoadWF.job(:prefix)
@@ -40,6 +55,81 @@ class TestSaveLoad < Test::Unit::TestCase
       newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
       assert_equal job.rec_dependencies.last.path, newjob.path
     end
+  end
+
+  def test_save_all_normal
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    options = {
+      :float => 0.5,
+      :integer => 15,
+      :string => "STRING",
+      :select => "option",
+      :array => [0,1,2,3],
+      :tsv => tsv,
+    }
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        job = TestSaveLoadWF.job(:save_test, nil, options)
+        TmpFile.with_file do |directory|
+          Step.save_job_inputs(job, directory)
+          newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+          assert_equal job.path, newjob.path
+        end
+      end
+    end
+  end
+
+  def test_save_all_file
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        TmpFile.with_file do |input_file|
+          Path.setup(input_file)
+          options = {
+            :float => input_file.float,
+            :integer => input_file.integer,
+            :string => input_file.string,
+            :select => input_file.select,
+            :array => input_file.array,
+            :tsv => input_file.tsv_file,
+          }
+          job = TestSaveLoadWF.job(:save_test, nil, options)
+          TmpFile.with_file do |directory|
+            Step.save_job_inputs(job, directory)
+            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+            assert_equal job.path, newjob.path
+          end
+        end
+      end
+    end
+  end
+
+  def test_save_all_job_file
+    tsv = TSV.setup({}, "Key~Value#:type=:single")
+    tsv["key"] = "value"
+    Log.with_severity 0 do
+      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
+        TmpFile.with_file do |input_file|
+          Path.setup(input_file)
+          options = {
+            :float => input_file.float,
+            :integer => input_file.integer,
+            :string => input_file.string,
+            :select => input_file.select,
+            :array => TestSaveLoadWF.job(:list),
+            :binary => TestSaveLoadWF.job(:list).file('binary'),
+            :tsv => input_file.tsv_file,
+          }
+          job = TestSaveLoadWF.job(:save_test, nil, options)
+          TmpFile.with_file do |directory|
+            Step.save_job_inputs(job, directory)
+            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
+            assert_equal job.path, newjob.path
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.8
+  version: 5.33.9
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-14 00:00:00.000000000 Z
+date: 2022-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake