rbbt-util 5.34.27 → 5.35.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40a3d86f01180255239e05fc23309b2c798eb94ee4c213ab8699f7954c1f5346
-  data.tar.gz: 3357e23002f032213a6b4c01ef046527092c8e4bf9ebce96dfc714176c3cb239
+  metadata.gz: 5efc019e19e967e30d83a08aa3ed45debe5036dac45f644ec0c7195100112e08
+  data.tar.gz: 1b7f34c9cb59d5936762531df4e004728071a90f4fcf3a4dce009e58071d7ffa
 SHA512:
-  metadata.gz: 8454ec0e14dac9fbdb5fa00a2219248b5037c890289fb5b013d69d6dac63e869b5160620d1ac744392546bf58dc8acfb1afba5c9172f366ddae01fc6cc008810
-  data.tar.gz: c85e5b6defde582d6af2c08281dd659bd8f71ae9867d4258d2c96c9824b3d0fe7020472fc6ccec3e66f4f70228bdc3dea425756abd510ded61a7f59420b78b98
+  metadata.gz: e0282a5eb3e8ec139b2cfc60face60d9bbd83d3597291df4e37fe9b8c4b2bca78a5fac23eb0c2ed4948d7cc61005c0625d992faaa67e265cd12443ee509f3037
+  data.tar.gz: a1e058254ad80b23409bb8f4618f14949089d6fbb355228fbf7abf82919d55a4ee86f47cdb1684014079a00f65831a5ea1900e63808fe21e863de3c67e121e86

data/lib/rbbt/hpc/batch.rb

@@ -378,17 +378,17 @@ function batch_sync_contain_dir(){
         prepare_environment +=<<-EOF
 # Load singularity modules
 command -v singularity &> /dev/null || module load singularity
-mkdir -p "#{singularity_opt_dir}"
+mkdir -p "#{File.expand_path singularity_opt_dir}"
         EOF
 
         if contain && options[:hardened]
 
           prepare_environment +=<<-EOF
 # Prepare container for singularity
-mkdir -p "#{contain}/.rbbt/etc/"
+mkdir -p "#{contain}"/.rbbt/etc/
 
 for dir in .ruby_inline git home; do
-    mkdir -p "#{contain}/$dir"
+    mkdir -p "#{contain}"/$dir
 done
 
 for tmpd in persist_locks  produce_locks  R_sockets  sensiblewrite  sensiblewrite_locks  step_info_locks  tsv_open_locks; do

data/lib/rbbt/hpc/orchestrate.rb

@@ -36,8 +36,16 @@ module HPC
       Log.high "Prepare for exec"
       prepare_for_execution(job)
 
-      if options[:orchestration_rules]
-        rules = Misc.load_yaml(options[:orchestration_rules])
+      if orchestration_rules_file = options[:orchestration_rules]
+        if Open.exists?(orchestration_rules_file)
+          rules = Misc.load_yaml(orchestration_rules_file)
+        elsif Rbbt.etc.slurm(orchestration_rules_file).exists?
+          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file))
+        elsif Rbbt.etc.slurm(orchestration_rules_file + '.yaml').exists?
+          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file + '.yaml'))
+        else
+          raise "Orchestration rules file not found: #{options[:orchestration_rules]}"
+        end
       elsif Rbbt.etc.slurm["default.yaml"].exists?
         rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
       else

data/lib/rbbt/resource.rb

@@ -90,6 +90,7 @@ module Resource
       
       lock_filename = nil # it seems like this was locked already.
 
+      Log.low "Downloading #{path} from #{url} file server"
       Misc.lock lock_filename do
         begin
           uri = URI(url)

data/lib/rbbt/tsv/attach/util.rb

@@ -105,7 +105,7 @@ module TSV
                   if other.include? source_key
                     v = case other.type
                         when :flat
-                          other[source_key] 
+                          other[source_key]
                         when :single
                           [other[source_key]]
                         when :double
@@ -135,8 +135,12 @@ module TSV
               if pos == :key
                 source_key
               else
-                if other.include? source_key
-                  v = other[source_key][pos]
+                if other.include?(source_key)
+                  if other.type == :single
+                    v = other[source_key]
+                  else
+                    v = other[source_key][pos]
+                  end
                   Array === v ? v.first : v
                 else
                   nil
@@ -192,7 +196,7 @@ module TSV
                   end
                 end
                 new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
-                new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+                new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and other.type == :double
                 new_values.flatten! if type == :flat
                 all_new_values << new_values
               end

data/lib/rbbt/util/R/model.rb

@@ -42,7 +42,7 @@ module R
     def colClasses(tsv)
       return nil unless TSV === tsv
       "c('character', " << 
-      (tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : ":NA") } * ", ") <<
+      (tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : "NA") } * ", ") <<
       ")"
     end
 

data/lib/rbbt/util/R/plot.rb

@@ -47,6 +47,8 @@ module R
             else
               "'character'"
             end
+          when Time
+            "'Date'"
           when Symbol
             "'factor'"
           else

data/lib/rbbt/util/R.rb

@@ -131,7 +131,7 @@ source(interactive.script.file)
       "Inf"
     when nil
       "NULL"
-    when "NA"
+    when ":NA"
       "NA"
     when TSV
       "matrix(#{R.ruby2R object.values},dimnames=list(#{R.ruby2R object.keys}, #{R.ruby2R object.fields}))"

data/lib/rbbt/util/cmd.rb

@@ -112,6 +112,7 @@ module CMD
     no_wait    = options.delete(:no_wait)
     xvfb       = options.delete(:xvfb)
     bar        = options.delete(:progress_bar)
+    save_stderr = options.delete(:save_stderr)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -154,7 +155,7 @@ module CMD
                                   Open3.popen3(ENV, cmd)
                                 rescue
                                   Log.warn $!.message
-                                  raise ProcessFailed, cmd unless no_fail
+                                  raise ProcessFailed, nil, cmd unless no_fail
                                   return
                                 end
     pid = wait_thr.pid
@@ -198,6 +199,7 @@ module CMD
           while line = serr.gets
             bar.process(line) if bar
             sout.log = line
+            sout.std_err << line if save_stderr
             Log.log "STDERR [#{pid}]: " +  line, stderr if log
           end 
           serr.close
@@ -244,9 +246,9 @@ module CMD
       status = wait_thr.value
       if not status.success? and not no_fail
         if !err.empty?
-          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+          raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
         else
-          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
+          raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
         end
       else
         Log.log err, stderr if Integer === stderr and log

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -7,7 +7,7 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
 
   def self.setup(stream, options = {}, &block)
     
@@ -20,6 +20,7 @@ module ConcurrentStream
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin unless autojoin.nil?
     stream.no_fail = no_fail unless no_fail.nil?
+    stream.std_err = ""
 
     stream.pair = pair unless pair.nil?
 
@@ -81,12 +82,16 @@ module ConcurrentStream
         begin
           t.join
           if Process::Status === t.value
-            if log
-              raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}. Last log line: #{log}" if ! (t.value.success? || no_fail)
-            else
-              raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}" if ! (t.value.success? || no_fail)
+            if ! (t.value.success? || no_fail)
+
+              if log
+                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+              else
+                msg = "Error joining #{self.filename || self.inspect}"
+              end
+
+              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
             end
-            raise ProcessFailed.new "Error joining process #{t.pid} in #{self.filename || self.inspect}. Last log line: #{log}" if ! (t.value.success? || no_fail)
           end
         rescue Exception
           if no_fail
@@ -106,11 +111,7 @@ module ConcurrentStream
       @pids.each do |pid| 
         begin
           Process.waitpid(pid, Process::WUNTRACED)
-          if log
-            stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}. Last log line: #{log}" unless $?.success? or no_fail
-          else
-            stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
-          end
+          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
       end 

data/lib/rbbt/util/misc/development.rb

@@ -420,7 +420,8 @@ def self.add_libdir(dir=nil)
   def self.ssh_run(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
-    CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
+    #CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
+    CMD.cmd("ssh '#{server}' ruby", :in => script, :log => true).read
   end
 
   def self.ssh_connection(server, reset = false)
@@ -437,7 +438,7 @@ def self.add_libdir(dir=nil)
                                   end
   end
 
-  def self.ssh_run(server, script = nil)
+  def self.ssh_run_alt(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
     write, master, pid = ssh_connection(server)

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,16 +1,39 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
 
+class MissingParameterException < ParameterException
+  def initialize(parameter)
+    super("Missing parameter '#{parameter}'")
+  end
+end
+
 class FieldNotFoundError < StandardError;end
 class ClosedStream < StandardError; end
 
 class ProcessFailed < StandardError; 
-  def initialize(pid = Process.pid)
+  attr_accessor :pid, :msg
+  def initialize(pid = Process.pid, msg = nil)
     @pid = pid
-    @msg = "Process #{@pid} failed"
-    super(@msg)
+    @msg = msg
+    if @pid
+      if @msg
+        message = "Process #{@pid} failed - #{@msg}"
+      else
+        message = "Process #{@pid} failed"
+      end
+    else
+      message = "Failed to run #{@msg}"
+    end
+    super(message)
   end
+end
 
+class ConcurrentStreamProcessFailed < ProcessFailed
+  attr_accessor :concurrent_stream
+  def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)
+    super(pid, msg)
+    @concurrent_stream = concurrent_stream
+  end
 end
 
 class Aborted < StandardError; end

data/lib/rbbt/util/misc/inspect.rb

@@ -276,7 +276,7 @@ module Misc
 
   def self.step_file?(path)
     return true if defined?(Step) && Step === path.resource
-    return false unless path.include?('.files/')
+    return false unless path =~ /\.files(?:\/|$)/
     parts = path.split("/")
     job = parts.select{|p| p =~ /\.files$/}.first
     if job

data/lib/rbbt/util/simpleopt/get.rb

@@ -1,8 +1,11 @@
 module SOPT
   GOT_OPTIONS= IndiferentHash.setup({})
+  def self.current_options=(options)
+    @@current_options = options
+  end
   def self.consume(args = ARGV)
     i = 0
-    values = {}
+    @@current_options ||= {}
     while i < args.length do
       current = args[i]
       break if current == "--"
@@ -25,20 +28,20 @@ module SOPT
 
       if input_types[input] == :string
         value = args.delete_at(i) if value.nil?
-        values[input] = value
+        @@current_options[input] = value
       else
         if value.nil? and %w(F false FALSE no).include?(args[i])
           Log.warn "Boolean values are best specified as #{current}=[true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value" 
           value = args.delete_at(i)
         end
-        values[input] = %w(F false FALSE no).include?(value)? false : true
+        @@current_options[input] = %w(F false FALSE no).include?(value)? false : true
       end
     end
 
-    IndiferentHash.setup values
-    GOT_OPTIONS.merge! values
+    IndiferentHash.setup @@current_options
+    GOT_OPTIONS.merge! @@current_options
 
-    values
+    @@current_options
   end
   
   def self.get(opt_str)

data/lib/rbbt/workflow/integration/ansible.rb

@@ -48,7 +48,7 @@ module Ansible
   end
 
   def self.playbook(file, task = nil, options = {})
-    task = 'default' if task.nil?
+    task = task.to_sym if String === task
 
     workflow = Workflow === file ? file : Workflow.require_workflow(file)
     task = workflow.tasks.keys.last if workflow.tasks[task].nil?

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -85,7 +85,9 @@ job = wf.job(task, jobname, job_inputs)
 STDOUT.write res.to_json
       EOF
 
-      JSON.parse(Misc.ssh_run(server, script))
+      json = Misc.ssh_run(server, script)
+      Log.debug "JSON (#{ url }): #{json}"
+      JSON.parse(json)
     end
 
     def self.get_raw(url, params)
@@ -118,19 +120,40 @@ STDOUT.write job.path
       script += job_script(input_id, jobname)
       script +=<<-EOF
 ENV["RBBT_UPDATE"]="#{(ENV["RBBT_UPDATE"] || false).to_s}"
-job.produce
+job.clean if job.error? and job.recoverable_error?
+job.run unless job.done? || job.error?
 STDOUT.write job.path
       EOF
       Misc.ssh_run(server, script)
     end
 
-    def self.run_slurm_job(url, input_id, jobname = nil)
+    def self.run_slurm_job(url, input_id, jobname = nil, slurm_options = {})
       server, path = parse_url(url)
 
       script = path_script(path)
       script += job_script(input_id, jobname)
       script +=<<-EOF
-job.produce
+require 'rbbt/hpc'
+HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+slurm_options = JSON.parse(%q(#{slurm_options.to_json}))
+job.clean if job.error? and job.recoverable_error?
+HPC::BATCH_MODULE.run_job(job, slurm_options) unless job.done? || job.error?
+STDOUT.write job.path
+      EOF
+      Misc.ssh_run(server, script)
+    end
+
+    def self.orchestrate_slurm_job(url, input_id, jobname = nil, slurm_options = {})
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script += job_script(input_id, jobname)
+      script +=<<-EOF
+require 'rbbt/hpc'
+HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+slurm_options = JSON.parse(%q(#{slurm_options.to_json}))
+job.clean if job.error? and job.recoverable_error?
+HPC::BATCH_MODULE.orchestrate_job(job, slurm_options) unless job.done? || job.error?
 STDOUT.write job.path
       EOF
       Misc.ssh_run(server, script)
@@ -190,18 +213,20 @@ job.clean
       produce = true if migrate
 
       workflow_name = job.workflow.to_s
-      inputs = job.inputs.to_hash
+      inputs = job.recursive_inputs.to_hash
+
       job.dependencies.each do |dep|
-        dep.produce if options[:produce_dependencies]
-        next unless dep.done?
+        dep.produce 
+      end if options[:produce_dependencies]
 
-        Step.migrate(dep.path, search_path, :target => server)
+      job.rec_dependencies.each do |dep|
+        Step.migrate(dep.path, search_path, :target => server) if dep.done?
       end
 
       remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
       rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
 
-      override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+      override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
       rjob.override_dependencies = override_dependencies
 
       if options[:migrate]

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -50,15 +50,20 @@ class RemoteStep
         "ssh://" + @server + ":" + ["var/jobs", self.workflow.to_s, task_name.to_s, @name] * "/"
       end
     end
+    
+    def _run
+      RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
+    end
 
     def produce(*args)
       input_types = {}
       init_job
-      @remote_path = RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
+      @remote_path = _run
       @started = true
-      while ! done?
+      while ! (done? || error?)
         sleep 1
       end
+      raise self.get_exception if error?
       self
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -266,6 +266,8 @@ class RemoteStep < Step
       end
     end
 
+    raise "Error in remote step" if self.error?
+
     self
   end
 

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -201,6 +201,7 @@ class Step
 
   def self.save_inputs(inputs, input_types, dir)
     inputs.each do |name,value|
+      next if value.nil?
       type = input_types[name]
       type = type.to_s if type
 

data/lib/rbbt/workflow.rb

@@ -520,6 +520,7 @@ module Workflow
   end
 
   def job(taskname, jobname = nil, inputs = {})
+    inputs, jobname = jobname, nil if Hash === jobname
     begin
       _job(taskname, jobname, inputs)
     ensure
@@ -858,7 +859,7 @@ module Workflow
   end
 
   def self.load_relay_tasks(filename)
-    remote_workflow_tasks = Misc.load_yaml(filename)
+    relay_workflow_tasks = Misc.load_yaml(filename)
     Workflow.process_relay_tasks(relay_workflow_tasks)
   end
 end

data/share/install/software/lib/install_helpers

@@ -85,7 +85,7 @@ uncompress_pkg(){
     mkdir -p "$OPT_BUILD_DIR"
     cd "$OPT_BUILD_DIR"
 
-    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & rmdir "$OPT_BUILD_DIR" & exit -1 ) ) 2> /dev/null
+    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & exit -1 ) ) 2> /dev/null
 
     cd "$old_pwd"
 }
@@ -178,7 +178,7 @@ clean_build(){
 }
 
 build_dir(){
-    echo $OPT_BUILD_DIR/`ls $OPT_BUILD_DIR |head -n 1`
+    echo $OPT_BUILD_DIR/$(ls $OPT_BUILD_DIR |head -n 1)
 }
 
 compile(){
@@ -211,6 +211,7 @@ prepare_make(){
     local extra="$@"
 
     local old_pwd="`expand_path $(pwd)`"
+
     cd "`build_dir`"
 
     [ -f bootstrap ] && (./bootstrap || exit -1)

data/share/rbbt_commands/ansible

@@ -42,7 +42,6 @@ if ! Misc.filename?(playbook_file)
   playbook_file = Ansible.playbook recipe, task, options
 end
 
-
 if dry_run
   if Hash === playbook_file
     ppp playbook_file.to_yaml

data/share/rbbt_commands/hpc/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/orchestrate

@@ -3,10 +3,13 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'rbbt/hpc/orchestrate'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -46,6 +49,8 @@ EOF
 batch_system = $slurm_options.delete :batch_system
 batch_system ||= 'auto'
 
+$slurm_options[:config_keys] = SOPT::GOT_OPTIONS[:config_keys]
+
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
@@ -61,7 +66,6 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
         @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
@@ -75,6 +79,14 @@ class Step
   end
 end
 
-ARGV.concat ['--detach']
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/task

@@ -3,9 +3,12 @@
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
 require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
 require 'rbbt/hpc'
 require 'time'
 
+rbbt_options = SOPT::GOT_OPTIONS
+
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
@@ -56,12 +59,20 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
       rescue HPC::SBATCH
       end
     end
   end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/workflow/task

@@ -547,7 +547,8 @@ if job_file = options.delete(:job_file)
 end
 
 case res
-when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
+#when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
+when (defined?(RemoteStep) and RemoteStep)
   res = job.result
   if res.respond_to? :gets
     begin
@@ -598,7 +599,7 @@ when Step
   elsif detach
     exit! 0
   else
-      res.join if res.running?
+    res.join if res.running?
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else

data/test/rbbt/hpc/test_batch.rb

@@ -6,7 +6,7 @@ Workflow.require_workflow "Sample"
 Workflow.require_workflow "HTS"
 class TestSLURM < Test::Unit::TestCase
 
-  def _test_batch_options
+  def test_batch_options
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -17,7 +17,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template
+  def test_template
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -28,7 +28,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template_singularity
+  def test_template_singularity
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|
@@ -39,7 +39,7 @@ class TestSLURM < Test::Unit::TestCase
     end
   end
 
-  def _test_template_contain
+  def test_template_contain
     job = Sample.job(:mutect2, "small", :reference => "hg38")
 
     TmpFile.with_file do |batch_dir|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.27
+  version: 5.35.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-15 00:00:00.000000000 Z
+date: 2023-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake