rbbt-util 5.37.16 → 5.38.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 565fc431e80f7167a95a8ad2c3499207c1422dacfe7ce1cd4cd2c1a4f0bad5c4
-  data.tar.gz: 34b393012cd4d58bcb9917cb2c45f8e0f23cefca73139647454b6affcb80cf57
+  metadata.gz: df9e238ba28f6bc0e16d1a22d8c9c34f7b57334f795ad9c4104ee1e689bc8fd5
+  data.tar.gz: 61cc2c91be510161edde982567750f13abb2388cf624e15ca1e4c72633382a3b
 SHA512:
-  metadata.gz: 854c7a84a92a654a987f5d1274dabd6d7564108d8a200eeb377321aa02361264d979f42f8e91350be2ed0074bd0f153b6dd66d7ea5f4597edca5b68edf2d8c0a
-  data.tar.gz: 34c765a22fed51dfaf873a0abcd0338f2a9ab15ed09bca0ac445075261ea040002e042312e62951c951bfc48d08162175cf697f9890e5b1a0c3ac58f2b0a03d0
+  metadata.gz: a3528b56bfc0faebf62c701fcc637a43af8ffe0639a795859f56dca57749b60a3cd821941b7e7bf079cca27a9a8aa8d9d69f71235b41c1ff48e73b4292a31683
+  data.tar.gz: dd71f1e7b233fb46d5e1210466bd6f64b0dd9162633d475f7e38bc8a057c853411bf27c35ffb3ce935f10b6e8d0055efdd7f5822adf09ff8f95a0bfa0efe8971

data/lib/rbbt/hpc/batch.rb

@@ -556,23 +556,32 @@ env > #{batch_options[:fenv]}
       fcmd
     end
 
+    def batch_dir_for_id(batch_base_dir, id)
+      job_id_file = Dir.glob(File.join(batch_base_dir, '*/job.id')).select{|f| Open.read(f).strip == id.to_s }.first
+      job_id_file ? File.dirname(job_id_file) : nil
+    end
 
     def run_job(job, options = {})
       system = self.to_s.split("::").last
 
+      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
+        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
+        :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
+
       if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
         Log.info "Job #{job.short_path} already queued in #{batch_job}"
-        return batch_job 
+        return batch_job, batch_dir_for_id(batch_base_dir, batch_job) 
       end
 
       if job.running?
         Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
-        return job.info[:batch_job]
-      end
 
-      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
-        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
-        :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
+        if job.info[:batch_job]
+          return job.info[:batch_job], batch_dir_for_id(batch_base_dir, batch_job)
+        else
+          return 
+        end
+      end
 
       workflow = job.original_workflow ||job.workflow
       task_name = job.original_task_name || job.task_name

data/lib/rbbt/hpc/orchestrate.rb

@@ -103,6 +103,7 @@ module HPC
           last_dir = dir
         end
       end
+
       [last_id, last_dir]
     end
 

data/lib/rbbt/resource/path.rb

@@ -479,4 +479,5 @@ module Path
   def clean_annotations
     "" << self.to_s
   end
+
 end

data/lib/rbbt/resource/rake.rb

@@ -53,7 +53,7 @@ module Rake
           Rake::Task.clear
           Rake::FileTask.clear_files
         end
-      rescue Exception
+      rescue 
         Log.error "Error in rake: #{$!.message}"
         Log.exception $!
         Kernel.exit! -1

data/lib/rbbt/util/misc/development.rb

@@ -423,13 +423,20 @@ def self.add_libdir(dir=nil)
     end
   end
 
-  def self.ssh_run(server, script = nil)
+  def self.ssh_run_old(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
     #CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
     CMD.cmd("ssh '#{server}' ruby", :in => script, :log => true).read
   end
 
+  def self.ssh_run(server, script = nil)
+    require 'rbbt/util/ssh'
+    Log.debug "Run ssh script in #{server}:\n#{script}"
+
+    SSHLine.ruby(server, script)
+  end
+
   def self.ssh_connection(server, reset = false)
     @@ssh_connections ||= {}
     @@ssh_connections.delete server if reset

data/lib/rbbt/util/misc/exceptions.rb

@@ -28,6 +28,16 @@ class ProcessFailed < StandardError;
   end
 end
 
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end
+
 class ConcurrentStreamProcessFailed < ProcessFailed
   attr_accessor :concurrent_stream
   def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)

data/lib/rbbt/util/ssh.rb

@@ -1,25 +1,76 @@
-module RbbtSSH
+require 'net/ssh'
 
-  def self.ssh(server, argv = nil, options = {})
-    server = server.sub(%r(^ssh:(//)?), '')
+class SSHLine
 
-    argv = [] if argv.nil?
-    argv = [argv] unless Array === argv
+  def initialize(host, user = nil)
+    @host = host
+    @user = user
 
-    options = Misc.add_defaults options, :add_option_dashes => true
+    @ssh = Net::SSH.start(@host, @user)
 
-    cmd_sections = [server]
-    cmd_sections << argv * " "
-    cmd_sections << CMD.process_cmd_options(options)
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash'
+    end
 
-    cmd = cmd_sections.compact * " "
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
 
-    CMD.cmd(:ssh, cmd, :pipe => true)
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
   end
 
-  def self.command(server, command, argv = [], options = nil)
-    ssh(server, [command] + argv, options)
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
   end
 
-end
+  def serve_output
+    @complete_output = true
+  end
+
+  def cmd(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
 
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||= SSHLine.new host, user
+  end
+
+  def self.run(server, cmd)
+    open(server).cmd(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
+end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -193,7 +193,8 @@ job.clean
           new = Step.migrate(paths, :user, :target => server)
           files.zip(new).each{|file,new| Open.write(file, new) }
 
-          CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
+          SSHLine.run(server, "mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/")
+          CMD.cmd_log("scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end
     end
@@ -220,21 +221,40 @@ job.clean
     #  rjob.run
     #end
 
-    def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
+    def self.upload_dependencies(job_list, server, search_path = 'user', produce_dependencies = false)
       server, path = parse_url(server) if server =~ /^ssh:\/\//
-      job.dependencies.each do |dep|
-        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
-        dep.produce
-      end if produce_dependencies
 
-      job.input_dependencies.each do |dep|
-        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
-        dep.produce
+      job_list = [job] unless Array === job_list
+
+      all_deps = {}
+      if produce_dependencies
+        job_list.each do |job|
+          job.dependencies.each do |dep|
+            all_deps[dep] ||= []
+            all_deps[dep] << job
+          end
+        end
+      end
+
+      job_list.each do |job|
+        job.input_dependencies.each do |dep|
+          all_deps[dep] ||= []
+          all_deps[dep] << job
+        end
+      end
+      iif all_deps
+
+      missing_deps = []
+      all_deps.each do |dep,jobs|
+        next if dep.done?
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Misc.fingerprint jobs}"
+        dep.run(true)
+        missing_deps << dep
       end 
+      Step.wait_for_jobs missing_deps
 
-      migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
-      migrate_dependencies += job.input_dependencies.select{|d| d.done? }.collect{|d| d.path }
-      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{job.path} to #{ server }" 
+      migrate_dependencies = all_deps.keys.select{|d| d.done? }.collect{|d| d.path }
+      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
 
@@ -242,7 +262,6 @@ job.clean
       inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
       job.dependencies.each do |dep|
         next if dep.done?
-        iif [dep, dep.inputs, dep.real_inputs]
         inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
         inputs = missing_dep_inputs(dep).merge(inputs)
       end
@@ -279,6 +298,47 @@ job.clean
       rjob
     end
 
+    def self.relay_job_list(job_list, server, options = {})
+      migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
+        :migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
+
+      search_path ||= 'user'
+
+      produce = true if migrate
+
+      upload_dependencies(job_list, server, search_path, options[:produce_dependencies])
+
+      rjobs_job = job_list.collect do |job|
+        
+        workflow_name = job.workflow.to_s
+        remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
+        inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+        Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
+
+        rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
+
+        override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+        rjob.override_dependencies = override_dependencies
+
+        rjob.run_type = run_type
+        rjob.slurm_options = slurm_options || {}
+
+        rjob.run(true)
+
+        [rjob, job]
+      end
+
+      if options[:migrate]
+        rjobs_job.each do |rjob,job|
+          rjob.produce
+          Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
+        end
+      end
+
+      rjobs_job.collect{|p| p.first }
+    end
+
+
     def self.relay(workflow, task, jobname, inputs, server, options = {})
       job = workflow.job(task, jobname, inputs)
       relay_job(job, server, options)

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -64,7 +64,7 @@ class RemoteStep
       RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
     end
 
-    def produce(*args)
+    def issue
       input_types = {}
       init_job
       @remote_path = case @run_type
@@ -76,6 +76,10 @@ class RemoteStep
                        _orchestrate_slurm
                      end
       @started = true
+    end
+
+    def produce(*args)
+      issue
       while ! (done? || error? || aborted?)
         sleep 1
       end
@@ -87,9 +91,13 @@ class RemoteStep
       load_res Open.open(path)
     end
 
-    def run(*args)
-      produce(*args)
-      self.load unless args.first
+    def run(stream = nil)
+      if stream
+        issue
+      else
+        produce
+        self.load unless args.first
+      end
     end
 
     def clean

data/lib/rbbt/workflow/step/run.rb

@@ -247,7 +247,7 @@ class Step
           @exec = false
           init_info(true)
 
-          workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
+          #workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
           result_type        = @task.respond_to?(:result_type)            ?  @task.result_type : nil
           result_description = @task.respond_to?(:result_description)     ?  @task.result_description : nil
 

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -67,9 +67,9 @@ module Workflow
           inputs[input.to_sym] = steps.first
         when :step_file
           path = Open.read(file).strip
-          path.extend Path
-          step_path = path.match(/(.*)\.files/)[1]
-          path.resource = Step.new step_path
+          step_path, relative = path.match(/(.*)\.files\/(.*)/).values_at 1, 2
+          step = Step.new Path.setup(step_path).find
+          path = step.file(relative)
           inputs[input.to_sym] = path
         when :step_file_array
           paths = Open.read(file).split("\n")
@@ -149,6 +149,8 @@ class Step
     case value
     when Path
       if Step === value.resource
+        step = value.resource
+        value = File.join('var/jobs', step.workflow.to_s, step.short_path + '.files', Misc.path_relative_to(step.files_dir, value))
         path = path + '.as_step_file'
       else
         path = path + '.as_path'

data/lib/rbbt/workflow/step.rb

@@ -57,7 +57,7 @@ class Step
   end
 
   def workflow
-    @workflow || info[:workflow]
+    @workflow || info[:workflow] || (@task && @task.respond_to?(:workflow) && @task.workflow) || path.split("/")[-3]
   end
 
 

data/lib/rbbt/workflow/usage.rb

@@ -128,7 +128,7 @@ module Workflow
       break if child
 
       if child
-        description << "-> " << task_name.to_s
+        description << "->" << task_name.to_s
       elsif first
         description << "" << task_name.to_s
       else
@@ -201,10 +201,10 @@ module Workflow
         description = description.split("\n\n").first
 
         next if abridge && ! final.include?(name)
-        puts Misc.format_definition_list_item(name.to_s, description, Log.tty_size || 80, 20, :yellow)
+        puts Misc.format_definition_list_item(name.to_s, description, Log.tty_size || 80, 30, :yellow)
 
         prov_string = prov_string(dep_tree(name))
-        puts Misc.format_paragraph Log.color(:blue, "-> " + prov_string) if prov_string && ! prov_string.empty?
+        puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
       end
 
     else