rbbt-util 5.34.26 → 5.35.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da3b496fd0fb7d9ff645cba0fb4ba3914288fb0b88bf80f43a79d02bd1ee2cfd
-  data.tar.gz: ab120f9bb96f1b0628f1edc6dc114a43c32ed39a5c29ce5252bbf85383d24359
+  metadata.gz: 5efc019e19e967e30d83a08aa3ed45debe5036dac45f644ec0c7195100112e08
+  data.tar.gz: 1b7f34c9cb59d5936762531df4e004728071a90f4fcf3a4dce009e58071d7ffa
 SHA512:
-  metadata.gz: 3f5b5326a834f1f972756919ea2f04c9626998be3c9ca058fd386c06787011ad021798e6091eb8eae1f7873dcceb9adcfa02de14a5269dadef4a2902d5475a9d
-  data.tar.gz: d74344affb899a6b62a494d02e9058dc338a7be4138b9ce32331d0657bc121035b312da21f48827751dc13e1bdee4511fb6f1eeaf481d39f73e5967ff3243800
+  metadata.gz: e0282a5eb3e8ec139b2cfc60face60d9bbd83d3597291df4e37fe9b8c4b2bca78a5fac23eb0c2ed4948d7cc61005c0625d992faaa67e265cd12443ee509f3037
+  data.tar.gz: a1e058254ad80b23409bb8f4618f14949089d6fbb355228fbf7abf82919d55a4ee86f47cdb1684014079a00f65831a5ea1900e63808fe21e863de3c67e121e86

data/lib/rbbt/hpc/batch.rb

@@ -378,17 +378,17 @@ function batch_sync_contain_dir(){
         prepare_environment +=<<-EOF
 # Load singularity modules
 command -v singularity &> /dev/null || module load singularity
-mkdir -p "#{singularity_opt_dir}"
+mkdir -p "#{File.expand_path singularity_opt_dir}"
         EOF
 
         if contain && options[:hardened]
 
           prepare_environment +=<<-EOF
 # Prepare container for singularity
-mkdir -p "#{contain}/.rbbt/etc/"
+mkdir -p "#{contain}"/.rbbt/etc/
 
 for dir in .ruby_inline git home; do
-    mkdir -p "#{contain}/$dir"
+    mkdir -p "#{contain}"/$dir
 done
 
 for tmpd in persist_locks  produce_locks  R_sockets  sensiblewrite  sensiblewrite_locks  step_info_locks  tsv_open_locks; do
@@ -560,6 +560,16 @@ env > #{batch_options[:fenv]}
     def run_job(job, options = {})
       system = self.to_s.split("::").last
 
+      if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
+        Log.info "Job #{job.short_path} already queued in #{batch_job}"
+        return batch_job 
+      end
+
+      if job.running?
+        Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
+        return job.info[:batch_job]
+      end
+
       batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
         :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
         :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
@@ -583,6 +593,8 @@ env > #{batch_options[:fenv]}
 
         batch_job = run_template(batch_dir, dry_run)
 
+        hold_dependencies(job, batch_job) unless dry_run
+
         return [batch_job, batch_dir] unless tail
 
         t_monitor = Thread.new do
@@ -603,6 +615,17 @@ env > #{batch_options[:fenv]}
       end
     end
 
+    def hold_dependencies(job, batch_job)
+      job.set_info :batch_job, batch_job
+      job.set_info :batch_system, self.batch_system
+      job.dependencies.each do |dep|
+        next unless dep.waiting?
+        next if (dep_batch_job = dep.info[:batch_job]) && job_queued(dep_batch_job)
+
+        hold_dependencies(dep, batch_job)
+      end
+    end
+
     def follow_job(batch_dir, tail = true)
       fjob = File.join(batch_dir, 'job.id')
       fout = File.join(batch_dir, 'std.out')
@@ -672,7 +695,7 @@ env > #{batch_options[:fenv]}
           terr = Misc.consume_stream(err, true, STDERR) if err
           tout = Misc.consume_stream(out, true, STDOUT) if out
 
-          sleep 3 while job_status(job).include? job.to_s
+          sleep 3 while job_queued(job)
         rescue Aborted
         ensure
           begin
@@ -692,6 +715,10 @@ env > #{batch_options[:fenv]}
       end
     end
 
+    def job_queued(job)
+      job_status(job).split(/\s+/).include?(job.to_s)
+    end
+
     def wait_for_job(batch_dir, time = 1)
       fexit = File.join(batch_dir, 'exit.status')
       fjob = File.join(batch_dir, 'job.id')

data/lib/rbbt/hpc/lsf.rb

@@ -5,6 +5,10 @@ module HPC
     extend HPC::TemplateGeneration
     extend HPC::Orchestration
 
+    def self.batch_system
+      "LSF"
+    end
+
     def self.batch_system_variables
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -13,7 +17,7 @@ let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") /
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
 export BATCH_JOB_ID=$LSF_JOBID
-export BATCH_SYSTEM=LSF
+export BATCH_SYSTEM=#{batch_system}
       EOF
     end
 

data/lib/rbbt/hpc/orchestrate.old.rb

@@ -206,7 +206,7 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules = Misc.load_yaml(options[:orchestration_rules]) if options[:orchestration_rules]
       rules ||= {}
       IndiferentHash.setup(rules)
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -36,10 +36,18 @@ module HPC
       Log.high "Prepare for exec"
       prepare_for_execution(job)
 
-      if options[:orchestration_rules]
-        rules = YAML.load(Open.read(options[:orchestration_rules]))
+      if orchestration_rules_file = options[:orchestration_rules]
+        if Open.exists?(orchestration_rules_file)
+          rules = Misc.load_yaml(orchestration_rules_file)
+        elsif Rbbt.etc.slurm(orchestration_rules_file).exists?
+          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file))
+        elsif Rbbt.etc.slurm(orchestration_rules_file + '.yaml').exists?
+          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file + '.yaml'))
+        else
+          raise "Orchestration rules file not found: #{options[:orchestration_rules]}"
+        end
       elsif Rbbt.etc.slurm["default.yaml"].exists?
-        rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
+        rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
       else
         rules = {}
       end

data/lib/rbbt/hpc/slurm.rb

@@ -6,6 +6,10 @@ module HPC
     extend HPC::TemplateGeneration
     extend HPC::Orchestration
 
+    def self.batch_system
+      "SLURM"
+    end
+
     def self.batch_system_variables
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -16,7 +20,7 @@ MAX_MEMORY="$MAX_MEMORY_DEFAULT"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
 export BATCH_JOB_ID=$SLURM_JOB_ID
-export BATCH_SYSTEM=SLURM
+export BATCH_SYSTEM=#{batch_system}
       EOF
     end
 
@@ -145,7 +149,11 @@ export BATCH_SYSTEM=SLURM
       if job.nil?
         CMD.cmd("squeue").read
       else
-        CMD.cmd("squeue --job #{job}").read
+        begin
+          CMD.cmd("squeue --job #{job}").read
+        rescue
+          ""
+        end
       end
     end
 

data/lib/rbbt/monitor.rb

@@ -64,14 +64,14 @@ module Rbbt
         lock_info[f].merge!(file_time(f))
         if File.size(f) > 0
           info = Open.open(f) do |s|
-            YAML.load(s)
+            Misc.load_yaml(s)
           end
           IndiferentHash.setup(info)
           lock_info[f][:pid] = info[:pid]
           lock_info[f][:ppid] = info[:ppid]
         end
       rescue Exception
-        #Log.exception $!
+        Log.exception $!
       end
     end
     lock_info
@@ -272,7 +272,7 @@ module Rbbt
   def self.load_lock(lock)
     begin
       info = Misc.insist 3 do
-        YAML.load(Open.read(lock))
+        Misc.load_yaml(lock)
       end
       info.values_at "pid", "ppid", "time"
     rescue Exception

data/lib/rbbt/persist.rb

@@ -148,9 +148,7 @@ module Persist
           JSON.parse(stream.read)
         end
       when :yaml
-        Open.open(path) do |stream|
-          YAML.load(stream)
-        end
+        Misc.load_yaml(path)
       when :float
         Open.read(path).to_f
       when :integer

data/lib/rbbt/resource/path.rb

@@ -150,7 +150,7 @@ module Path
   search_path_file = File.join(ENV['HOME'], '.rbbt/etc/search_paths')
   if File.exist?(search_path_file)
     begin
-      YAML.load(File.open(search_path_file)).each do |where, location|
+      Misc.load_yaml(search_path_file).each do |where, location|
         SEARCH_PATHS[where.to_sym] = location
       end
     rescue

data/lib/rbbt/resource.rb

@@ -90,6 +90,7 @@ module Resource
       
       lock_filename = nil # it seems like this was locked already.
 
+      Log.low "Downloading #{path} from #{url} file server"
       Misc.lock lock_filename do
         begin
           uri = URI(url)

data/lib/rbbt/tsv/attach/util.rb

@@ -105,7 +105,7 @@ module TSV
                   if other.include? source_key
                     v = case other.type
                         when :flat
-                          other[source_key] 
+                          other[source_key]
                         when :single
                           [other[source_key]]
                         when :double
@@ -135,8 +135,12 @@ module TSV
               if pos == :key
                 source_key
               else
-                if other.include? source_key
-                  v = other[source_key][pos]
+                if other.include?(source_key)
+                  if other.type == :single
+                    v = other[source_key]
+                  else
+                    v = other[source_key][pos]
+                  end
                   Array === v ? v.first : v
                 else
                   nil
@@ -192,7 +196,7 @@ module TSV
                   end
                 end
                 new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
-                new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+                new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and other.type == :double
                 new_values.flatten! if type == :flat
                 all_new_values << new_values
               end

data/lib/rbbt/util/R/model.rb

@@ -42,7 +42,7 @@ module R
     def colClasses(tsv)
       return nil unless TSV === tsv
       "c('character', " << 
-      (tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : ":NA") } * ", ") <<
+      (tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : "NA") } * ", ") <<
       ")"
     end
 

data/lib/rbbt/util/R/plot.rb

@@ -47,6 +47,8 @@ module R
             else
               "'character'"
             end
+          when Time
+            "'Date'"
           when Symbol
             "'factor'"
           else

data/lib/rbbt/util/R.rb

@@ -131,7 +131,7 @@ source(interactive.script.file)
       "Inf"
     when nil
       "NULL"
-    when "NA"
+    when ":NA"
       "NA"
     when TSV
       "matrix(#{R.ruby2R object.values},dimnames=list(#{R.ruby2R object.keys}, #{R.ruby2R object.fields}))"

data/lib/rbbt/util/cmd.rb

@@ -112,6 +112,7 @@ module CMD
     no_wait    = options.delete(:no_wait)
     xvfb       = options.delete(:xvfb)
     bar        = options.delete(:progress_bar)
+    save_stderr = options.delete(:save_stderr)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -154,7 +155,7 @@ module CMD
                                   Open3.popen3(ENV, cmd)
                                 rescue
                                   Log.warn $!.message
-                                  raise ProcessFailed, cmd unless no_fail
+                                  raise ProcessFailed, nil, cmd unless no_fail
                                   return
                                 end
     pid = wait_thr.pid
@@ -198,6 +199,7 @@ module CMD
           while line = serr.gets
             bar.process(line) if bar
             sout.log = line
+            sout.std_err << line if save_stderr
             Log.log "STDERR [#{pid}]: " +  line, stderr if log
           end 
           serr.close
@@ -244,9 +246,9 @@ module CMD
       status = wait_thr.value
       if not status.success? and not no_fail
         if !err.empty?
-          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+          raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
         else
-          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
+          raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
         end
       else
         Log.log err, stderr if Integer === stderr and log

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -7,7 +7,7 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
 
   def self.setup(stream, options = {}, &block)
     
@@ -20,6 +20,7 @@ module ConcurrentStream
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin unless autojoin.nil?
     stream.no_fail = no_fail unless no_fail.nil?
+    stream.std_err = ""
 
     stream.pair = pair unless pair.nil?
 
@@ -81,12 +82,16 @@ module ConcurrentStream
         begin
           t.join
           if Process::Status === t.value
-            if log
-              raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}. Last log line: #{log}" if ! (t.value.success? || no_fail)
-            else
-              raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}" if ! (t.value.success? || no_fail)
+            if ! (t.value.success? || no_fail)
+
+              if log
+                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+              else
+                msg = "Error joining #{self.filename || self.inspect}"
+              end
+
+              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
             end
-            raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}. Last log line: #{log}" if ! (t.value.success? || no_fail)
           end
         rescue Exception
           if no_fail
@@ -106,11 +111,7 @@ module ConcurrentStream
       @pids.each do |pid| 
         begin
           Process.waitpid(pid, Process::WUNTRACED)
-          if log
-            stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}. Last log line: #{log}" unless $?.success? or no_fail
-          else
-            stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
-          end
+          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
       end 

data/lib/rbbt/util/misc/development.rb

@@ -420,7 +420,8 @@ def self.add_libdir(dir=nil)
   def self.ssh_run(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
-    CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
+    #CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
+    CMD.cmd("ssh '#{server}' ruby", :in => script, :log => true).read
   end
 
   def self.ssh_connection(server, reset = false)
@@ -437,7 +438,7 @@ def self.add_libdir(dir=nil)
                                   end
   end
 
-  def self.ssh_run(server, script = nil)
+  def self.ssh_run_alt(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
     write, master, pid = ssh_connection(server)

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,16 +1,39 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
 
+class MissingParameterException < ParameterException
+  def initialize(parameter)
+    super("Missing parameter '#{parameter}'")
+  end
+end
+
 class FieldNotFoundError < StandardError;end
 class ClosedStream < StandardError; end
 
 class ProcessFailed < StandardError; 
-  def initialize(pid = Process.pid)
+  attr_accessor :pid, :msg
+  def initialize(pid = Process.pid, msg = nil)
     @pid = pid
-    @msg = "Process #{@pid} failed"
-    super(@msg)
+    @msg = msg
+    if @pid
+      if @msg
+        message = "Process #{@pid} failed - #{@msg}"
+      else
+        message = "Process #{@pid} failed"
+      end
+    else
+      message = "Failed to run #{@msg}"
+    end
+    super(message)
   end
+end
 
+class ConcurrentStreamProcessFailed < ProcessFailed
+  attr_accessor :concurrent_stream
+  def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)
+    super(pid, msg)
+    @concurrent_stream = concurrent_stream
+  end
 end
 
 class Aborted < StandardError; end

data/lib/rbbt/util/misc/inspect.rb

@@ -276,7 +276,7 @@ module Misc
 
   def self.step_file?(path)
     return true if defined?(Step) && Step === path.resource
-    return false unless path.include?('.files/')
+    return false unless path =~ /\.files(?:\/|$)/
     parts = path.split("/")
     job = parts.select{|p| p =~ /\.files$/}.first
     if job

data/lib/rbbt/util/misc/serialize.rb

@@ -0,0 +1,26 @@
+module Misc
+  def self.load_yaml(yaml)
+    case yaml
+    when IO, StringIO
+      if YAML.respond_to?(:unsafe_load)
+        YAML.unsafe_load(yaml)
+      else
+        YAML.load(yaml)
+      end
+    when (defined?(Path) && Path)
+      yaml.open do |io|
+        load_yaml(io)
+      end
+    when String
+      if Misc.is_filename?(yaml)
+        File.open(yaml) do |io|
+          load_yaml(io)
+        end
+      else
+        load_yaml(StringIO.new(yaml))
+      end
+    else
+      raise "Unkown YAML object: #{Misc.fingerprint yaml}"
+    end
+  end
+end

data/lib/rbbt/util/misc.rb

@@ -22,6 +22,8 @@ require 'rbbt/util/misc/objects'
 require 'rbbt/util/misc/manipulation'
 require 'rbbt/util/misc/communication'
 
+require 'rbbt/util/misc/serialize'
+
 require 'to_regexp'
 
 module MultipleResult; end

data/lib/rbbt/util/simpleopt/get.rb

@@ -1,8 +1,11 @@
 module SOPT
   GOT_OPTIONS= IndiferentHash.setup({})
+  def self.current_options=(options)
+    @@current_options = options
+  end
   def self.consume(args = ARGV)
     i = 0
-    values = {}
+    @@current_options ||= {}
     while i < args.length do
       current = args[i]
       break if current == "--"
@@ -25,20 +28,20 @@ module SOPT
 
       if input_types[input] == :string
         value = args.delete_at(i) if value.nil?
-        values[input] = value
+        @@current_options[input] = value
       else
         if value.nil? and %w(F false FALSE no).include?(args[i])
           Log.warn "Boolean values are best specified as #{current}=[true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value" 
           value = args.delete_at(i)
         end
-        values[input] = %w(F false FALSE no).include?(value)? false : true
+        @@current_options[input] = %w(F false FALSE no).include?(value)? false : true
       end
     end
 
-    IndiferentHash.setup values
-    GOT_OPTIONS.merge! values
+    IndiferentHash.setup @@current_options
+    GOT_OPTIONS.merge! @@current_options
 
-    values
+    @@current_options
   end
   
   def self.get(opt_str)

data/lib/rbbt/workflow/integration/ansible.rb

@@ -48,7 +48,7 @@ module Ansible
   end
 
   def self.playbook(file, task = nil, options = {})
-    task = 'default' if task.nil?
+    task = task.to_sym if String === task
 
     workflow = Workflow === file ? file : Workflow.require_workflow(file)
     task = workflow.tasks.keys.last if workflow.tasks[task].nil?

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -85,7 +85,9 @@ job = wf.job(task, jobname, job_inputs)
 STDOUT.write res.to_json
       EOF
 
-      JSON.parse(Misc.ssh_run(server, script))
+      json = Misc.ssh_run(server, script)
+      Log.debug "JSON (#{ url }): #{json}"
+      JSON.parse(json)
     end
 
     def self.get_raw(url, params)
@@ -118,19 +120,40 @@ STDOUT.write job.path
       script += job_script(input_id, jobname)
       script +=<<-EOF
 ENV["RBBT_UPDATE"]="#{(ENV["RBBT_UPDATE"] || false).to_s}"
-job.produce
+job.clean if job.error? and job.recoverable_error?
+job.run unless job.done? || job.error?
 STDOUT.write job.path
       EOF
       Misc.ssh_run(server, script)
     end
 
-    def self.run_slurm_job(url, input_id, jobname = nil)
+    def self.run_slurm_job(url, input_id, jobname = nil, slurm_options = {})
       server, path = parse_url(url)
 
       script = path_script(path)
       script += job_script(input_id, jobname)
       script +=<<-EOF
-job.produce
+require 'rbbt/hpc'
+HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+slurm_options = JSON.parse(%q(#{slurm_options.to_json}))
+job.clean if job.error? and job.recoverable_error?
+HPC::BATCH_MODULE.run_job(job, slurm_options) unless job.done? || job.error?
+STDOUT.write job.path
+      EOF
+      Misc.ssh_run(server, script)
+    end
+
+    def self.orchestrate_slurm_job(url, input_id, jobname = nil, slurm_options = {})
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script += job_script(input_id, jobname)
+      script +=<<-EOF
+require 'rbbt/hpc'
+HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+slurm_options = JSON.parse(%q(#{slurm_options.to_json}))
+job.clean if job.error? and job.recoverable_error?
+HPC::BATCH_MODULE.orchestrate_job(job, slurm_options) unless job.done? || job.error?
 STDOUT.write job.path
       EOF
       Misc.ssh_run(server, script)
@@ -190,18 +213,20 @@ job.clean
       produce = true if migrate
 
       workflow_name = job.workflow.to_s
-      inputs = job.inputs.to_hash
+      inputs = job.recursive_inputs.to_hash
+
       job.dependencies.each do |dep|
-        dep.produce if options[:produce_dependencies]
-        next unless dep.done?
+        dep.produce 
+      end if options[:produce_dependencies]
 
-        Step.migrate(dep.path, search_path, :target => server)
+      job.rec_dependencies.each do |dep|
+        Step.migrate(dep.path, search_path, :target => server) if dep.done?
       end
 
       remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
       rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
 
-      override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+      override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
       rjob.override_dependencies = override_dependencies
 
       if options[:migrate]

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -50,15 +50,20 @@ class RemoteStep
         "ssh://" + @server + ":" + ["var/jobs", self.workflow.to_s, task_name.to_s, @name] * "/"
       end
     end
+    
+    def _run
+      RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
+    end
 
     def produce(*args)
       input_types = {}
       init_job
-      @remote_path = RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
+      @remote_path = _run
       @started = true
-      while ! done?
+      while ! (done? || error?)
         sleep 1
       end
+      raise self.get_exception if error?
       self
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -266,6 +266,8 @@ class RemoteStep < Step
       end
     end
 
+    raise "Error in remote step" if self.error?
+
     self
   end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -143,8 +143,12 @@ class Step
 
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
+    batch_job = info[:batch_job] if Open.exists?(info_file)
+    batch_system = info[:batch_system] if Open.exists?(info_file)
     Open.lock(info_file, :lock => info_lock) do
       i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
+      i[:batch_job] = batch_job if batch_job
+      i[:batch_system] = batch_system if batch_system
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -253,7 +257,7 @@ class Step
       #Open.read(file(name)).split /\n|,\s*/
       Open.read(file(name)).split "\n"
     when :yaml
-      YAML.load(Open.open(file(name)))
+      Misc.load_yaml(file(name))
     when :marshal
       Marshal.load(Open.open(file(name)))
     else

data/lib/rbbt/workflow/step/run.rb

@@ -698,7 +698,7 @@ class Step
   end
 
   def grace
-    until done? || result || error? || aborted? || streaming? || waiting? 
+    until done? || result || error? || aborted? || streaming? || waiting? || running?
       sleep 1 
     end
     self

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -82,7 +82,7 @@ module Workflow
         when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
-            inputs[input.to_sym]  = YAML.load(Open.read(file))
+            inputs[input.to_sym]  = Misc.load_yaml(file)
           else
             if File.symlink?(file)
               link_target = File.expand_path(File.readlink(file), File.dirname(file))
@@ -201,6 +201,7 @@ class Step
 
   def self.save_inputs(inputs, input_types, dir)
     inputs.each do |name,value|
+      next if value.nil?
       type = input_types[name]
       type = type.to_s if type
 

data/lib/rbbt/workflow/step/status.rb

@@ -15,7 +15,7 @@ class Step
       raise "DO NOT CLEAN" 
     end
 
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir) or Open.exists?(pid_file)
 
       @result = nil
       @pid = nil

data/lib/rbbt/workflow.rb

@@ -520,6 +520,7 @@ module Workflow
   end
 
   def job(taskname, jobname = nil, inputs = {})
+    inputs, jobname = jobname, nil if Hash === jobname
     begin
       _job(taskname, jobname, inputs)
     ensure
@@ -834,8 +835,7 @@ module Workflow
   end
 
   def self.load_remote_tasks(filename)
-    yaml_text = Open.read(filename)
-    remote_workflow_tasks = YAML.load(yaml_text)
+    remote_workflow_tasks = Misc.load_yaml(filename)
     Workflow.process_remote_tasks(remote_workflow_tasks)
   end
 
@@ -859,8 +859,7 @@ module Workflow
   end
 
   def self.load_relay_tasks(filename)
-    yaml_text = Open.read(filename)
-    relay_workflow_tasks = YAML.load(yaml_text)
+    relay_workflow_tasks = Misc.load_yaml(filename)
     Workflow.process_relay_tasks(relay_workflow_tasks)
   end
 end

data/share/install/software/lib/install_helpers

@@ -85,7 +85,7 @@ uncompress_pkg(){
     mkdir -p "$OPT_BUILD_DIR"
     cd "$OPT_BUILD_DIR"
 
-    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & rmdir "$OPT_BUILD_DIR" & exit -1 ) ) 2> /dev/null
+    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & exit -1 ) ) 2> /dev/null
 
     cd "$old_pwd"
 }
@@ -178,7 +178,7 @@ clean_build(){
 }
 
 build_dir(){
-    echo $OPT_BUILD_DIR/`ls $OPT_BUILD_DIR |head -n 1`
+    echo $OPT_BUILD_DIR/$(ls $OPT_BUILD_DIR |head -n 1)
 }
 
 compile(){
@@ -211,6 +211,7 @@ prepare_make(){
     local extra="$@"
 
     local old_pwd="`expand_path $(pwd)`"
+
     cd "`build_dir`"
 
     [ -f bootstrap ] && (./bootstrap || exit -1)

data/share/rbbt_commands/ansible

@@ -42,7 +42,6 @@ if ! Misc.filename?(playbook_file)
   playbook_file = Ansible.playbook recipe, task, options
 end
 
-
 if dry_run
   if Hash === playbook_file
     ppp playbook_file.to_yaml

data/share/rbbt_commands/hpc/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/system/clean

@@ -46,7 +46,7 @@ if locks.any?
   puts
   puts Log.color(:magenta, "Locks:")
   locks.each do |file,info|
-    if force or (info[:pid] and not Misc.pid_exists? info[:pid])
+    if force or (info[:pid] && ! Misc.pid_exists?(info[:pid]))
       puts "  Removing #{ file }"
       File.unlink file 
     end

data/share/rbbt_commands/workflow/task

@@ -547,7 +547,8 @@ if job_file = options.delete(:job_file)
 end
 
 case res
-when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
+#when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
+when (defined?(RemoteStep) and RemoteStep)
   res = job.result
   if res.respond_to? :gets
     begin
@@ -598,7 +599,7 @@ when Step
   elsif detach
     exit! 0
   else
-      res.join if res.running?
+    res.join if res.running?
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else

data/test/rbbt/hpc/test_batch.rb

@@ -6,7 +6,7 @@ Workflow.require_workflow "Sample"
 Workflow.require_workflow "HTS"
 class TestSLURM < Test::Unit::TestCase
 
-  def _test_batch_options
+  def test_batch_options
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -17,7 +17,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template
+  def test_template
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -28,7 +28,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template_singularity
+  def test_template_singularity
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -39,7 +39,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template_contain
+  def test_template_contain
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|

data/test/rbbt/util/misc/test_serialize.rb

@@ -0,0 +1,24 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/serialize'
+
+class TestClass < Test::Unit::TestCase
+  def test_load_yaml
+    yaml_txt=<<-EOF
+---
+a: b
+    EOF
+    yaml_sio = StringIO.new yaml_txt
+
+    assert_equal "b", Misc.load_yaml(yaml_txt)["a"]
+    assert_equal "b", Misc.load_yaml(yaml_sio)["a"]
+
+    TmpFile.with_file yaml_txt do |yaml_file|
+      assert_equal "b", Misc.load_yaml(yaml_file)["a"]
+      Open.open(yaml_file) do |yaml_io|
+        assert_equal "b", Misc.load_yaml(yaml_io)["a"]
+      end
+    end
+
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.26
+  version: 5.35.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-14 00:00:00.000000000 Z
+date: 2023-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -323,6 +323,7 @@ files:
 - lib/rbbt/util/misc/omics.rb
 - lib/rbbt/util/misc/options.rb
 - lib/rbbt/util/misc/pipes.rb
+- lib/rbbt/util/misc/serialize.rb
 - lib/rbbt/util/misc/system.rb
 - lib/rbbt/util/named_array.rb
 - lib/rbbt/util/open.rb
@@ -559,6 +560,7 @@ files:
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/python/test_util.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -634,6 +636,7 @@ test_files:
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_lock.rb