rbbt-util 5.37.16 → 5.38.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 565fc431e80f7167a95a8ad2c3499207c1422dacfe7ce1cd4cd2c1a4f0bad5c4
-  data.tar.gz: 34b393012cd4d58bcb9917cb2c45f8e0f23cefca73139647454b6affcb80cf57
+  metadata.gz: 1121593dec25c29d7986047727d555ed0e75df6b158f9f1332764ad4886462c5
+  data.tar.gz: c8e39ef65e23445cf5d0c8817af961a9206016c5e3665d87e042f0fadec2b8df
 SHA512:
-  metadata.gz: 854c7a84a92a654a987f5d1274dabd6d7564108d8a200eeb377321aa02361264d979f42f8e91350be2ed0074bd0f153b6dd66d7ea5f4597edca5b68edf2d8c0a
-  data.tar.gz: 34c765a22fed51dfaf873a0abcd0338f2a9ab15ed09bca0ac445075261ea040002e042312e62951c951bfc48d08162175cf697f9890e5b1a0c3ac58f2b0a03d0
+  metadata.gz: 22c87766544a52c9c6f0d6da369422a452f553d1a61b82a54674f676a3867a6c1d29028c2efd031563f982cdadbce56d36950c8e100bd34ac2a7e95b39af8a58
+  data.tar.gz: 0a79edc89dad388bca9b94dfeccba1df64c4294e84c4f19743608038e55f8800181911d787980adae3ec1c715a993430ccbf2bc07d6212485b89843ada93de74

data/lib/rbbt/annotations.rb

@@ -154,6 +154,8 @@ module Annotated
 
   def self.purge(object)
     case object
+    when Path
+      object
     when String
       object.respond_to?(:clean_annotations) ?
         object.clean_annotations : 

data/lib/rbbt/hpc/batch.rb

@@ -556,23 +556,32 @@ env > #{batch_options[:fenv]}
       fcmd
     end
 
+    def batch_dir_for_id(batch_base_dir, id)
+      job_id_file = Dir.glob(File.join(batch_base_dir, '*/job.id')).select{|f| Open.read(f).strip == id.to_s }.first
+      job_id_file ? File.dirname(job_id_file) : nil
+    end
 
     def run_job(job, options = {})
       system = self.to_s.split("::").last
 
+      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
+        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
+        :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
+
       if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
         Log.info "Job #{job.short_path} already queued in #{batch_job}"
-        return batch_job 
+        return batch_job, batch_dir_for_id(batch_base_dir, batch_job) 
       end
 
       if job.running?
         Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
-        return job.info[:batch_job]
-      end
 
-      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
-        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
-        :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
+        if job.info[:batch_job]
+          return job.info[:batch_job], batch_dir_for_id(batch_base_dir, batch_job)
+        else
+          return 
+        end
+      end
 
       workflow = job.original_workflow ||job.workflow
       task_name = job.original_task_name || job.task_name

data/lib/rbbt/hpc/orchestrate.rb

@@ -103,6 +103,7 @@ module HPC
           last_dir = dir
         end
       end
+
       [last_id, last_dir]
     end
 

data/lib/rbbt/persist.rb

@@ -115,7 +115,7 @@ module Persist
     begin
       case (type || :marshal).to_sym
       when :path
-        path
+        Path.setup(Open.read(path).strip)
       when :nil
         nil
       when :boolean
@@ -172,7 +172,7 @@ module Persist
 
     case (type || :marshal).to_sym
     when :path
-      nil
+      Open.write(path, content)
     when :nil
       nil
     when :boolean

data/lib/rbbt/resource/path.rb

@@ -479,4 +479,5 @@ module Path
   def clean_annotations
     "" << self.to_s
   end
+
 end

data/lib/rbbt/resource/rake.rb

@@ -53,7 +53,7 @@ module Rake
           Rake::Task.clear
           Rake::FileTask.clear_files
         end
-      rescue Exception
+      rescue 
         Log.error "Error in rake: #{$!.message}"
         Log.exception $!
         Kernel.exit! -1

data/lib/rbbt/resource.rb

@@ -350,6 +350,7 @@ url='#{url}'
   end
 
   def identify(path)
+    return path unless path.start_with?("/")
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
     resource ||= Rbbt
@@ -357,6 +358,7 @@ url='#{url}'
     locations -= [:current, "current"]
     locations << :current
     search_paths = IndiferentHash.setup(resource.search_paths)
+    choices = []
     locations.uniq.each do |name|
       pattern = search_paths[name]
       pattern = resource.search_paths[pattern] while Symbol === pattern
@@ -370,12 +372,15 @@ url='#{url}'
         "(?:/(?<REST>.*))?/?$"
         if m = path.match(regexp) 
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
-            return self[m["TOPLEVEL"]][m["SUBPATH"]][m["REST"]]
+            unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
+            unlocated.gsub!(/\/+/,'/')
+            unlocated[self.subdir] = "" if self.subdir
+            choices << self.annotate(unlocated)
           end
         end
       end
     end
-    nil
+    choices.sort_by{|s| s.length }.first
   end
 end
 

data/lib/rbbt/tsv/excel.rb

@@ -154,7 +154,11 @@ module TSV
       sheet1 = book.create_worksheet 
       sheet1.name = sheet if sheet
 
-      sheet1.row(0).concat fields
+      if fields
+        sheet1.row(0).concat fields if fields
+      else
+        sheet1.row(0).concat ["No field info"]
+      end
 
       rows.each_with_index do |cells,i|
         sheet1.row(i+1).concat cells

data/lib/rbbt/tsv/parser.rb

@@ -130,12 +130,12 @@ module TSV
       keys = parts[key_position].split(@sep2, -1)
       values = case
                when field_positions.nil?
-                parts.tap{|o| o.delete_at key_position}
+                 parts.tap{|o| o.delete_at key_position}
                when field_positions.empty?
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| (value.nil? || value.empty?) ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.nil? || value.empty?) ? [""] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/util/misc/development.rb

@@ -423,13 +423,18 @@ def self.add_libdir(dir=nil)
     end
   end
 
-  def self.ssh_run(server, script = nil)
+  def self.ssh_run_old(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
     #CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
     CMD.cmd("ssh '#{server}' ruby", :in => script, :log => true).read
   end
 
+  def self.ssh_run(server, script = nil)
+    require 'rbbt/util/ssh'
+    SSHLine.ruby(server, script)
+  end
+
   def self.ssh_connection(server, reset = false)
     @@ssh_connections ||= {}
     @@ssh_connections.delete server if reset

data/lib/rbbt/util/misc/exceptions.rb

@@ -28,6 +28,16 @@ class ProcessFailed < StandardError;
   end
 end
 
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end
+
 class ConcurrentStreamProcessFailed < ProcessFailed
   attr_accessor :concurrent_stream
   def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)

data/lib/rbbt/util/misc/format.rb

@@ -27,6 +27,9 @@ module Misc
 
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
+    offset = 0 if offset.nil?
+    indent = 0 if indent.nil?
+    size = 80 if size.nil?
     size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
       text.split(re).collect do |paragraph|

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -22,18 +22,21 @@ module IndiferentHash
     @_default ||= self.default or self.default_proc
   end
 
-  def [](key)
-    res = super(key) 
+  def [](key, *args)
+    res = super(key, *args) 
+    res = IndiferentHash.setup(res) if Hash === res
     return res unless res.nil? or (_default? and not keys.include? key)
 
     case key
     when Symbol, Module
-      super(key.to_s)
+      res = super(key.to_s, *args)
     when String
-      super(key.to_sym)
-    else
-      res
+      res = super(key.to_sym, *args)
     end
+
+    res = IndiferentHash.setup(res) if Hash === res
+
+    res
   end
 
   def values_at(*key_list)

data/lib/rbbt/util/misc/inspect.rb

@@ -275,7 +275,6 @@ module Misc
 
 
   def self.step_file?(path)
-    return true if defined?(Step) && Step === path.resource
     return false unless path =~ /\.files(?:\/|$)/
     parts = path.split("/")
     job = parts.select{|p| p =~ /\.files$/}.first
@@ -283,9 +282,19 @@ module Misc
       i = parts.index job
       begin
         workflow, task = parts.values_at i - 2, i - 1
-        Workflow.require_workflow workflow
+        _loaded = false
+        begin
+          Kernel.const_get(workflow)
+        rescue 
+          if ! _loaded
+            Workflow.require_workflow workflow 
+            _loaded = true
+            retry
+          end
+          raise $!
+        end
         #return Kernel.const_get(workflow).tasks.include? task.to_sym
-        return true
+        return parts[i-2..-1] * "/"
       rescue
         Log.exception $!
       end
@@ -293,6 +302,8 @@ module Misc
     false
   end
 
+  RBBT_IDENTIFY_PATH = ENV["RBBT_IDENTIFY_PATH"] != 'false'
+
   def self.obj2str(obj)
     _obj = obj
     obj = Annotated.purge(obj) if Annotated === obj
@@ -310,18 +321,21 @@ module Misc
             'false'
           when Hash
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
-          when (defined?(Path) and Path)
+          when (defined?(Path) && Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
-            elsif step_file?(obj)
-              "Step file: " + obj
+            elsif step_file_path = step_file?(obj)
+              "Step file: " + step_file_path
             else
               if obj.exists?
+                obj = (obj.resource || Rbbt).identify obj if RBBT_IDENTIFY_PATH
                 if obj.directory?
                   files = obj.glob("**/*")
-                  "directory: #{Misc.fingerprint(files)}"
-                else
+                  "directory: #{Misc.fingerprint(files.collect{|f| Misc.path_relative_to(obj.find, f)}.sort)}"
+                elsif obj.located?
                   "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
+                else
+                  "path: " << obj 
                 end
               else
                 obj + " (file missing)"

data/lib/rbbt/util/misc/options.rb

@@ -111,7 +111,7 @@ module Misc
     html = if content.nil?
       "<#{ tag }#{attr_str}/>"
     else
-      "<#{ tag }#{attr_str}>#{ content }</#{ tag }>"
+      "<#{ tag }#{attr_str}>#{ content.to_s }</#{ tag }>"
     end
 
     html

data/lib/rbbt/util/misc/pipes.rb

@@ -318,9 +318,9 @@ module Misc
           Open.mkdir dir unless Open.exists?(dir)
           into_path, into = into, Open.open(into, :mode => 'w') 
         end
-        into.sync = true if IO === into
+        #into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
-        io.sync = true
+        #io.sync = true
 
         begin
           while c = io.readpartial(BLOCK_SIZE)
@@ -402,7 +402,7 @@ module Misc
           when (IO === content or StringIO === content or File === content)
 
             Open.write(tmp_path) do |f|
-              f.sync = true
+              #f.sync = true
               while block = content.read(BLOCK_SIZE)
                 f.write block
               end 

data/lib/rbbt/util/open.rb

@@ -240,7 +240,6 @@ module Open
   end
 
   def self.find_repo_dir(file)
-    return nil
     self.repository_dirs.each do |dir|
       dir = dir + '/' unless dir.chars[-1] == "/"
 

data/lib/rbbt/util/ssh.rb

@@ -1,25 +1,118 @@
-module RbbtSSH
+require 'net/ssh'
 
-  def self.ssh(server, argv = nil, options = {})
-    server = server.sub(%r(^ssh:(//)?), '')
+class SSHLine
 
-    argv = [] if argv.nil?
-    argv = [argv] unless Array === argv
+  def initialize(host, user = nil)
+    @host = host
+    @user = user
 
-    options = Misc.add_defaults options, :add_option_dashes => true
+    @ssh = Net::SSH.start(@host, @user)
 
-    cmd_sections = [server]
-    cmd_sections << argv * " "
-    cmd_sections << CMD.process_cmd_options(options)
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash'
+    end
 
-    cmd = cmd_sections.compact * " "
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
 
-    CMD.cmd(:ssh, cmd, :pipe => true)
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
   end
 
-  def self.command(server, command, argv = [], options = nil)
-    ssh(server, [command] + argv, options)
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
   end
 
-end
+  def serve_output
+    @complete_output = true
+  end
+
+  def run(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
+
+  def rbbt(script)
+    rbbt_script =<<-EOF
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+  res = begin
+      old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+#{script}
+      ensure
+        STDOUT.reopen(old_stdout)
+      end
+
+  puts Marshal.dump(res)
+    EOF
+
+    m = ruby(rbbt_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    rbbt(preamble + "\n" + script)
+  end
+
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||= SSHLine.new host, user
+  end
+
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
 
+  def self.rbbt(server, script)
+    open(server).rbbt(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    run(server, [command] + argv, options)
+  end
+end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -99,7 +99,7 @@ STDOUT.write res.to_json
       JSON.parse(json)
     end
 
-    def self.get_raw(url, params)
+    def self.get_raw(url, params = {})
       server, path = parse_url(url)
       script = path_script(path)
 
@@ -193,7 +193,8 @@ job.clean
           new = Step.migrate(paths, :user, :target => server)
           files.zip(new).each{|file,new| Open.write(file, new) }
 
-          CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
+          SSHLine.run(server, "mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/")
+          CMD.cmd_log("scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end
     end
@@ -220,21 +221,39 @@ job.clean
     #  rjob.run
     #end
 
-    def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
+    def self.upload_dependencies(job_list, server, search_path = 'user', produce_dependencies = false)
       server, path = parse_url(server) if server =~ /^ssh:\/\//
-      job.dependencies.each do |dep|
-        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
-        dep.produce
-      end if produce_dependencies
 
-      job.input_dependencies.each do |dep|
-        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
-        dep.produce
-      end 
+      job_list = [job_list] unless Array === job_list
+
+      all_deps = {}
+      if produce_dependencies
+        job_list.each do |job|
+          job.dependencies.each do |dep|
+            all_deps[dep] ||= []
+            all_deps[dep] << job
+          end
+        end
+      end
+
+      job_list.each do |job|
+        job.input_dependencies.each do |dep|
+          all_deps[dep] ||= []
+          all_deps[dep] << job
+        end
+      end
 
-      migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
-      migrate_dependencies += job.input_dependencies.select{|d| d.done? }.collect{|d| d.path }
-      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{job.path} to #{ server }" 
+      missing_deps = []
+      all_deps.each do |dep,jobs|
+        next if dep.done?
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Misc.fingerprint jobs}"
+        dep.run(true)
+        missing_deps << dep
+      end if produce_dependencies
+      Step.wait_for_jobs missing_deps
+
+      migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
+      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
 
@@ -242,7 +261,6 @@ job.clean
       inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
       job.dependencies.each do |dep|
         next if dep.done?
-        iif [dep, dep.inputs, dep.real_inputs]
         inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
         inputs = missing_dep_inputs(dep).merge(inputs)
       end
@@ -279,6 +297,48 @@ job.clean
       rjob
     end
 
+    def self.relay_job_list(job_list, server, options = {})
+      migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
+        :migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
+
+      search_path ||= 'user'
+
+      produce = true if migrate
+
+      upload_dependencies(job_list, server, search_path, options[:produce_dependencies])
+
+      rjobs_job = job_list.collect do |job|
+        
+        workflow_name = job.workflow.to_s
+        remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
+        inputs = IndiferentHash.setup(job.recursive_inputs.to_hash).slice(*job.real_inputs.map{|i| i.to_s})
+        Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
+
+        rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
+
+        override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+        rjob.override_dependencies = override_dependencies
+
+        rjob.run_type = run_type
+        rjob.slurm_options = slurm_options || {}
+
+        rjob.run(true)
+
+        [rjob, job]
+      end
+
+      if options[:migrate]
+        rjobs_job.each do |rjob,job|
+          rjob.produce
+          iif [:migrate, job]
+          Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
+        end
+      end
+
+      rjobs_job.collect{|p| p.first }
+    end
+
+
     def self.relay(workflow, task, jobname, inputs, server, options = {})
       job = workflow.job(task, jobname, inputs)
       relay_job(job, server, options)