rbbt-util 5.38.0 → 5.38.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df9e238ba28f6bc0e16d1a22d8c9c34f7b57334f795ad9c4104ee1e689bc8fd5
-  data.tar.gz: 61cc2c91be510161edde982567750f13abb2388cf624e15ca1e4c72633382a3b
+  metadata.gz: 1121593dec25c29d7986047727d555ed0e75df6b158f9f1332764ad4886462c5
+  data.tar.gz: c8e39ef65e23445cf5d0c8817af961a9206016c5e3665d87e042f0fadec2b8df
 SHA512:
-  metadata.gz: a3528b56bfc0faebf62c701fcc637a43af8ffe0639a795859f56dca57749b60a3cd821941b7e7bf079cca27a9a8aa8d9d69f71235b41c1ff48e73b4292a31683
-  data.tar.gz: dd71f1e7b233fb46d5e1210466bd6f64b0dd9162633d475f7e38bc8a057c853411bf27c35ffb3ce935f10b6e8d0055efdd7f5822adf09ff8f95a0bfa0efe8971
+  metadata.gz: 22c87766544a52c9c6f0d6da369422a452f553d1a61b82a54674f676a3867a6c1d29028c2efd031563f982cdadbce56d36950c8e100bd34ac2a7e95b39af8a58
+  data.tar.gz: 0a79edc89dad388bca9b94dfeccba1df64c4294e84c4f19743608038e55f8800181911d787980adae3ec1c715a993430ccbf2bc07d6212485b89843ada93de74

data/lib/rbbt/annotations.rb

@@ -154,6 +154,8 @@ module Annotated
 
   def self.purge(object)
     case object
+    when Path
+      object
     when String
       object.respond_to?(:clean_annotations) ?
         object.clean_annotations : 

data/lib/rbbt/persist.rb

@@ -115,7 +115,7 @@ module Persist
     begin
       case (type || :marshal).to_sym
       when :path
-        path
+        Path.setup(Open.read(path).strip)
       when :nil
         nil
       when :boolean
@@ -172,7 +172,7 @@ module Persist
 
     case (type || :marshal).to_sym
     when :path
-      nil
+      Open.write(path, content)
     when :nil
       nil
     when :boolean

data/lib/rbbt/resource.rb

@@ -350,6 +350,7 @@ url='#{url}'
   end
 
   def identify(path)
+    return path unless path.start_with?("/")
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
     resource ||= Rbbt
@@ -357,6 +358,7 @@ url='#{url}'
     locations -= [:current, "current"]
     locations << :current
     search_paths = IndiferentHash.setup(resource.search_paths)
+    choices = []
     locations.uniq.each do |name|
       pattern = search_paths[name]
       pattern = resource.search_paths[pattern] while Symbol === pattern
@@ -370,12 +372,15 @@ url='#{url}'
         "(?:/(?<REST>.*))?/?$"
         if m = path.match(regexp) 
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
-            return self[m["TOPLEVEL"]][m["SUBPATH"]][m["REST"]]
+            unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
+            unlocated.gsub!(/\/+/,'/')
+            unlocated[self.subdir] = "" if self.subdir
+            choices << self.annotate(unlocated)
           end
         end
       end
     end
-    nil
+    choices.sort_by{|s| s.length }.first
   end
 end
 

data/lib/rbbt/tsv/excel.rb

@@ -154,7 +154,11 @@ module TSV
       sheet1 = book.create_worksheet 
       sheet1.name = sheet if sheet
 
-      sheet1.row(0).concat fields
+      if fields
+        sheet1.row(0).concat fields if fields
+      else
+        sheet1.row(0).concat ["No field info"]
+      end
 
       rows.each_with_index do |cells,i|
         sheet1.row(i+1).concat cells

data/lib/rbbt/tsv/parser.rb

@@ -130,12 +130,12 @@ module TSV
       keys = parts[key_position].split(@sep2, -1)
       values = case
                when field_positions.nil?
-                parts.tap{|o| o.delete_at key_position}
+                 parts.tap{|o| o.delete_at key_position}
                when field_positions.empty?
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| (value.nil? || value.empty?) ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.nil? || value.empty?) ? [""] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/util/misc/development.rb

@@ -432,8 +432,6 @@ def self.add_libdir(dir=nil)
 
   def self.ssh_run(server, script = nil)
     require 'rbbt/util/ssh'
-    Log.debug "Run ssh script in #{server}:\n#{script}"
-
     SSHLine.ruby(server, script)
   end
 

data/lib/rbbt/util/misc/format.rb

@@ -27,6 +27,9 @@ module Misc
 
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
+    offset = 0 if offset.nil?
+    indent = 0 if indent.nil?
+    size = 80 if size.nil?
     size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
       text.split(re).collect do |paragraph|

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -22,18 +22,21 @@ module IndiferentHash
     @_default ||= self.default or self.default_proc
   end
 
-  def [](key)
-    res = super(key) 
+  def [](key, *args)
+    res = super(key, *args) 
+    res = IndiferentHash.setup(res) if Hash === res
     return res unless res.nil? or (_default? and not keys.include? key)
 
     case key
     when Symbol, Module
-      super(key.to_s)
+      res = super(key.to_s, *args)
     when String
-      super(key.to_sym)
-    else
-      res
+      res = super(key.to_sym, *args)
     end
+
+    res = IndiferentHash.setup(res) if Hash === res
+
+    res
   end
 
   def values_at(*key_list)

data/lib/rbbt/util/misc/inspect.rb

@@ -275,7 +275,6 @@ module Misc
 
 
   def self.step_file?(path)
-    return true if defined?(Step) && Step === path.resource
     return false unless path =~ /\.files(?:\/|$)/
     parts = path.split("/")
     job = parts.select{|p| p =~ /\.files$/}.first
@@ -283,9 +282,19 @@ module Misc
       i = parts.index job
       begin
         workflow, task = parts.values_at i - 2, i - 1
-        Workflow.require_workflow workflow
+        _loaded = false
+        begin
+          Kernel.const_get(workflow)
+        rescue 
+          if ! _loaded
+            Workflow.require_workflow workflow 
+            _loaded = true
+            retry
+          end
+          raise $!
+        end
         #return Kernel.const_get(workflow).tasks.include? task.to_sym
-        return true
+        return parts[i-2..-1] * "/"
       rescue
         Log.exception $!
       end
@@ -293,6 +302,8 @@ module Misc
     false
   end
 
+  RBBT_IDENTIFY_PATH = ENV["RBBT_IDENTIFY_PATH"] != 'false'
+
   def self.obj2str(obj)
     _obj = obj
     obj = Annotated.purge(obj) if Annotated === obj
@@ -310,18 +321,21 @@ module Misc
             'false'
           when Hash
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
-          when (defined?(Path) and Path)
+          when (defined?(Path) && Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
-            elsif step_file?(obj)
-              "Step file: " + obj
+            elsif step_file_path = step_file?(obj)
+              "Step file: " + step_file_path
             else
               if obj.exists?
+                obj = (obj.resource || Rbbt).identify obj if RBBT_IDENTIFY_PATH
                 if obj.directory?
                   files = obj.glob("**/*")
-                  "directory: #{Misc.fingerprint(files)}"
-                else
+                  "directory: #{Misc.fingerprint(files.collect{|f| Misc.path_relative_to(obj.find, f)}.sort)}"
+                elsif obj.located?
                   "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
+                else
+                  "path: " << obj 
                 end
               else
                 obj + " (file missing)"

data/lib/rbbt/util/misc/options.rb

@@ -111,7 +111,7 @@ module Misc
     html = if content.nil?
       "<#{ tag }#{attr_str}/>"
     else
-      "<#{ tag }#{attr_str}>#{ content }</#{ tag }>"
+      "<#{ tag }#{attr_str}>#{ content.to_s }</#{ tag }>"
     end
 
     html

data/lib/rbbt/util/misc/pipes.rb

@@ -318,9 +318,9 @@ module Misc
           Open.mkdir dir unless Open.exists?(dir)
           into_path, into = into, Open.open(into, :mode => 'w') 
         end
-        into.sync = true if IO === into
+        #into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
-        io.sync = true
+        #io.sync = true
 
         begin
           while c = io.readpartial(BLOCK_SIZE)
@@ -402,7 +402,7 @@ module Misc
           when (IO === content or StringIO === content or File === content)
 
             Open.write(tmp_path) do |f|
-              f.sync = true
+              #f.sync = true
               while block = content.read(BLOCK_SIZE)
                 f.write block
               end 

data/lib/rbbt/util/open.rb

@@ -240,7 +240,6 @@ module Open
   end
 
   def self.find_repo_dir(file)
-    return nil
     self.repository_dirs.each do |dir|
       dir = dir + '/' unless dir.chars[-1] == "/"
 

data/lib/rbbt/util/ssh.rb

@@ -37,7 +37,7 @@ class SSHLine
     @complete_output = true
   end
 
-  def cmd(command)
+  def run(command)
     send_cmd(command)
     @ssh.loop{ ! @complete_output}
     if @exit_status.to_i == 0
@@ -51,6 +51,7 @@ class SSHLine
     @output = ""
     @complete_output = false
     cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
     @ch.send_data(cmd)
     @ch.send_data("echo DONECMD: $?\n")
     @ssh.loop{ !@complete_output }
@@ -61,16 +62,57 @@ class SSHLine
     end
   end
 
+  def rbbt(script)
+    rbbt_script =<<-EOF
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+  res = begin
+      old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+#{script}
+      ensure
+        STDOUT.reopen(old_stdout)
+      end
+
+  puts Marshal.dump(res)
+    EOF
+
+    m = ruby(rbbt_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    rbbt(preamble + "\n" + script)
+  end
+
   @connections = {}
   def self.open(host, user = nil)
     @connections[[host, user]] ||= SSHLine.new host, user
   end
 
-  def self.run(server, cmd)
-    open(server).cmd(cmd)
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
   end
 
   def self.ruby(server, script)
     open(server).ruby(script)
   end
+
+  def self.rbbt(server, script)
+    open(server).rbbt(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    run(server, [command] + argv, options)
+  end
 end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -99,7 +99,7 @@ STDOUT.write res.to_json
       JSON.parse(json)
     end
 
-    def self.get_raw(url, params)
+    def self.get_raw(url, params = {})
       server, path = parse_url(url)
       script = path_script(path)
 
@@ -224,7 +224,7 @@ job.clean
     def self.upload_dependencies(job_list, server, search_path = 'user', produce_dependencies = false)
       server, path = parse_url(server) if server =~ /^ssh:\/\//
 
-      job_list = [job] unless Array === job_list
+      job_list = [job_list] unless Array === job_list
 
       all_deps = {}
       if produce_dependencies
@@ -242,7 +242,6 @@ job.clean
           all_deps[dep] << job
         end
       end
-      iif all_deps
 
       missing_deps = []
       all_deps.each do |dep,jobs|
@@ -250,10 +249,10 @@ job.clean
         Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Misc.fingerprint jobs}"
         dep.run(true)
         missing_deps << dep
-      end 
+      end if produce_dependencies
       Step.wait_for_jobs missing_deps
 
-      migrate_dependencies = all_deps.keys.select{|d| d.done? }.collect{|d| d.path }
+      migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
       Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
@@ -312,7 +311,7 @@ job.clean
         
         workflow_name = job.workflow.to_s
         remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
-        inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+        inputs = IndiferentHash.setup(job.recursive_inputs.to_hash).slice(*job.real_inputs.map{|i| i.to_s})
         Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
 
         rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
@@ -331,6 +330,7 @@ job.clean
       if options[:migrate]
         rjobs_job.each do |rjob,job|
           rjob.produce
+          iif [:migrate, job]
           Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
         end
       end

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -1,6 +1,6 @@
 class RemoteStep
   module SSH
-    attr_accessor :override_dependencies, :run_type, :slurm_options
+    attr_accessor :override_dependencies, :run_type, :slurm_options, :produce_dependencies
 
     def init_job(cache_type = nil, other_params = {})
       return self if @url
@@ -52,11 +52,12 @@ class RemoteStep
     end
     
     def _run
-      RemoteWorkflow::SSH.upload_dependencies(self, @server)
+      RemoteWorkflow::SSH.upload_dependencies(self, @server, 'user', @produce_dependencies)
       RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
     end
 
     def _run_slurm
+      RemoteWorkflow::SSH.upload_dependencies(self, @server, 'user', @produce_dependencies)
       RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
     end
 
@@ -115,7 +116,7 @@ class RemoteStep
         select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
         inputs.values.flatten.
         select{|dep| Path === dep && Step === dep.resource }.
-        select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+        #select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
         collect{|dep| dep.resource }
     end
   end

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -188,6 +188,7 @@ class RemoteStep < Step
   end
 
   def file(file)
+    init_job
     @adaptor.get_raw(File.join(url, 'file', file.to_s))
   end
 

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,7 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise_dependency_error(job) if job.error? and not canfail
+    Step.raise_dependency_error(job) if job.error? and not canfail
   end
 
   def self.raise_dependency_error(job)
@@ -149,7 +149,7 @@ class Step
       select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
       recursive_inputs.flatten.
       select{|dep| Path === dep && Step === dep.resource }.
-      select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+      #select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
       collect{|dep| dep.resource }
   end
 
@@ -193,7 +193,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise_dependency_error dependency 
+        Step.raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -382,7 +382,7 @@ class Step
 
   def run_dependencies
 
-    rec_dependencies = self.rec_dependencies(true) + input_dependencies
+    rec_dependencies = self.rec_dependencies(true) + input_dependencies.reject{|d| d.started? }
 
     return if rec_dependencies.empty?
 
@@ -402,7 +402,7 @@ class Step
       seen_paths << step.path
 
       begin
-        Step.prepare_for_execution(step) unless step == self
+        Step.prepare_for_execution(step) unless step == self 
       rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
@@ -423,6 +423,7 @@ class Step
 
     produced = []
     (dependencies + input_dependencies).each do |dep|
+      next if dep.started?
       next unless ComputeDependency === dep
       if dep.compute == :produce
         dep.produce 

data/lib/rbbt/workflow/step/info.rb

@@ -132,7 +132,7 @@ class Step
       messages
     else
       set_info(:messages, []) if self.respond_to?(:set_info)
-    end
+    end || []
   end
 
   def message(message)

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -49,9 +49,9 @@ module Workflow
         when :nofile
           inputs[input.to_sym]  = Open.realpath(file)
         when :path_array
-          inputs[input.to_sym]  = Open.read(file).strip.split("\n")
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n").collect{|p| Path.setup(p) }
         when :path
-          inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
+          inputs[input.to_sym]  = Path.setup(Open.read(file).strip.split("\n").first)
         when :io
           inputs[input.to_sym] = Open.open(Open.realpath(file))
         when :io_array
@@ -156,7 +156,7 @@ class Step
         path = path + '.as_path'
       end
     when String
-      if Misc.is_filename?(value, false)
+      if Misc.is_filename?(value, true)
         value = value.dup
         value.extend Path
         return save_input(name, value, type, dir)
@@ -175,7 +175,7 @@ class Step
           path = path + '.as_path_array'
         end
       when String
-        if Misc.is_filename?(value.first, false)
+        if Misc.is_filename?(value.first, true)
           path = path + '.as_path_array'
         end
       when IO
@@ -251,5 +251,4 @@ class Step
 
     inputs.keys
   end
-
 end

data/lib/rbbt/workflow/step.rb

@@ -57,7 +57,12 @@ class Step
   end
 
   def workflow
-    @workflow || info[:workflow] || (@task && @task.respond_to?(:workflow) && @task.workflow) || path.split("/")[-3]
+    @workflow ||= begin
+                    wf = info[:workflow] 
+                    wf = nil if wf == ""
+                    wf ||= @task.workflow if @task && @task.respond_to?(:workflow)
+                    wf ||= path.split("/")[-3]
+                  end
   end
 
 
@@ -296,7 +301,7 @@ class Step
               res = @result
             else
               join if not done?
-              res = @path.exists? ? Persist.load_file(@path, result_type) : exec
+              res = @path.exists? ? Persist.load_file(@path, result_type) : run
             end
 
             if result_description

data/lib/rbbt/workflow.rb

@@ -486,7 +486,7 @@ module Workflow
         if dep_basename.include? "."
           parts = dep_basename.split(".")
           extension = [parts.pop]
-          while parts.last.length <= 4
+          while parts.length > 1 && parts.last.length <= 4
             extension << parts.pop
           end
           extension = extension.reverse * "."

data/share/rbbt_commands/resource/find

@@ -84,5 +84,5 @@ if where.nil? || where == 'all' || path.search_paths.include?(where.to_sym)
     puts location
   end
 else
-  puts RbbtSSH.command(where, $0, ARGV, options.merge("where" => :all)).read
+  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all))
 end

data/share/rbbt_commands/resource/glob

@@ -86,5 +86,5 @@ if where.nil? || where == 'all' || path.search_paths.include?(where.to_sym)
 
   puts location * "\n"
 else
-  puts RbbtSSH.command(where, $0, ARGV, options.merge("where" => :all)).read
+  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all))
 end

data/share/rbbt_commands/workflow/task

@@ -93,6 +93,8 @@ def fix_options(workflow, task, job_options)
               TrueClass === value or %w(true TRUE T yes).include? value
             when :float
               value.to_f
+            when :path
+              Path.setup(value)
             when :integer
               value.to_i
             when :text
@@ -419,13 +421,6 @@ begin
     exit 0
   end
 
-  if tasks = options.delete(:produce)
-    tasks = tasks.split(",")
-    produce_cpus = (options[:produce_cpus] || 1)
-    puts Step.produce_dependencies(job, tasks,  prepare_cpus)
-    exit 0
-  end
-
   def match_dependencies(queries, dependencies)
     queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
 
@@ -443,6 +438,50 @@ begin
     matched
   end
 
+  def replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay = false, run_type = :run)
+    jobs_to_relay.each do |job|
+      ComputeDependency.setup(job, :bootstrap)
+      next if job.done?
+      Log.low "Relaying #{Misc.fingerprint job} to #{server}"
+      jmeta = class << job; self; end
+
+      job.instance_variable_set(:@job, job)
+      job.instance_variable_set(:@host, server)
+      job.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
+
+      jmeta.define_method :run do |*args|
+        if done?
+          load
+        else
+          RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
+          Step.migrate(@job, 'user', :source => @host)
+          load
+        end
+      end
+      #job.dependencies = []
+
+      #([job] + job.rec_dependencies).each do |j|
+      #  next if job.done?
+      #  jmeta = class << j; self; end
+
+      #  j.instance_variable_set(:@job, job)
+      #  j.instance_variable_set(:@host, server)
+      #  j.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
+
+      #  jmeta.define_method :run do |*args|
+      #    if done?
+      #      load
+      #    else
+      #      RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
+      #      Step.migrate(@job, 'user', :source => @host)
+      #      load
+      #    end
+      #  end
+      #end
+    end
+  end
+
+
   if server = options.delete(:relay)
     require 'rbbt/workflow/remote_workflow'
     relay_dependencies = options.delete(:relay_dependencies).split(",")
@@ -451,7 +490,7 @@ begin
     jobs_to_relay = relay_dependencies ?  match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
     jobs_to_relay.reject!{|d| d.done? }
 
-    RemoteWorkflow::SSH.relay_job_list(jobs_to_relay, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :run)
   end
 
   if server = options.delete(:slurm_relay)
@@ -461,7 +500,7 @@ begin
     jobs_to_relay = relay_dependencies ?  match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
     jobs_to_relay.reject!{|d| d.done? }
 
-    RemoteWorkflow::SSH.relay_job_list(jobs_to_relay, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :orchestrate)
   end
 
 
@@ -483,6 +522,14 @@ begin
     end
   end
 
+  if tasks = options.delete(:produce)
+    tasks = tasks.split(",")
+    produce_cpus = (options[:produce_cpus] || 1)
+    puts Step.produce_dependencies(job, tasks,  produce_cpus)
+    exit 0
+  end
+
+
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
     if detach

data/test/rbbt/test_resource.rb

@@ -7,6 +7,8 @@ require 'test/unit'
 module TestResource
   extend Resource
 
+  self.subdir = 'tmp/tmp-share/resource/TestResource/'
+
   claim tmp.test.google, :url, "http://google.com"
   claim tmp.test.string, :string, "TEST"
   claim tmp.test.proc, :proc do
@@ -79,8 +81,8 @@ class TestTSV < Test::Unit::TestCase
   end
 
   def test_libdir
-    assert File.exist?(TestResource[].share.Rlib["util.R"].find(:lib))
-    assert File.exist?(TestResource[].share.Rlib["util.R"].find)
+    assert File.exist?(Rbbt.share.Rlib["util.R"].find(:lib))
+    assert File.exist?(Rbbt.share.Rlib["util.R"].find)
   end
 
   def __test_server
@@ -100,6 +102,9 @@ class TestTSV < Test::Unit::TestCase
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
+
+    assert_equal 'etc/somefile', TestResource.identify(TestResource.etc.somefile.find(:user))
+    assert_equal 'etc/somefile', TestResource.identify(TestResource.etc.somefile.find(:local))
   end
 
 end

data/test/rbbt/tsv/test_accessor.rb

@@ -199,13 +199,30 @@ row3    A|AA|AAA|AAA    B    Id3    3
 
   def test_unzip
     content =<<-EOF
-#Id    ValueA    ValueB    OtherID
-row1    a|A    b|B    Id1|Id2
-row2    aa|aa|AA|AA    b1|b2|B1|B2    Id1|Id1|Id2|Id2
+#: :type=:list
+#PMID:Sentence number:TF:TG	Transcription Factor (Associated Gene Name)	Target Gene (Associated Gene Name)	Sign	Negation	PMID
+24265317:3:NR1H3:FASN	NR1H3	FASN			24265317
+17522048:0:NR1H3:FASN	NR1H3	FASN	+		17522048
+19903962:0:NR1H3:FASN	NR1H3	FASN			19903962
+19903962:7:NR1H3:FASN	NR1H3	FASN			19903962
+22183856:4:NR1H3:FASN	NR1H3	FASN			22183856
+22641099:4:NR1H3:FASN	NR1H3	FASN	+		22641099
+23499676:8:NR1H3:FASN	NR1H3	FASN	+		23499676
+11790787:5:NR1H3:FASN	NR1H3	FASN			11790787
+11790787:7:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:9:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:11:NR1H3:FASN	NR1H3	FASN			11790787
+17522048:1:NR1H3:FASN	NR1H3	FASN	+		17522048
+17522048:3:NR1H3:FASN	NR1H3	FASN			17522048
+22160584:1:NR1H3:FASN	NR1H3	FASN			22160584
+22160584:5:NR1H3:FASN	NR1H3	FASN	+		22160584
+22160584:8:NR1H3:FASN	NR1H3	FASN	+		22160584
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/)
+      tsv = TSV.open(filename, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "PMID"], :merge => true, :type => :double)
+      unzip = tsv.unzip(0,true)
+      assert_equal unzip["NR1H3:FASN"][0].length, unzip["NR1H3:FASN"][1].length
     end
   end
 

data/test/rbbt/tsv/test_parser.rb

@@ -69,5 +69,33 @@ A B C D E
     end
   end
 
+  def test_merge
+    content =<<-EOF
+#: :type=:double
+#PMID:Sentence number:TF:TG	Transcription Factor (Associated Gene Name)	Target Gene (Associated Gene Name)	Sign	Negation	PMID
+24265317:3:NR1H3:FASN	NR1H3	FASN			24265317
+17522048:0:NR1H3:FASN	NR1H3	FASN	+		17522048
+19903962:0:NR1H3:FASN	NR1H3	FASN			19903962
+19903962:7:NR1H3:FASN	NR1H3	FASN			19903962
+22183856:4:NR1H3:FASN	NR1H3	FASN			22183856
+22641099:4:NR1H3:FASN	NR1H3	FASN	+		22641099
+23499676:8:NR1H3:FASN	NR1H3	FASN	+		23499676
+11790787:5:NR1H3:FASN	NR1H3	FASN			11790787
+11790787:7:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:9:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:11:NR1H3:FASN	NR1H3	FASN			11790787
+17522048:1:NR1H3:FASN	NR1H3	FASN	+		17522048
+17522048:3:NR1H3:FASN	NR1H3	FASN			17522048
+22160584:1:NR1H3:FASN	NR1H3	FASN			22160584
+22160584:5:NR1H3:FASN	NR1H3	FASN	+		22160584
+22160584:8:NR1H3:FASN	NR1H3	FASN	+		22160584
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "PMID"], :merge => true, :type => :double)
+      assert_equal 16, tsv["NR1H3"]["Sign"].length
+    end
+  end
+
 end
 

data/test/rbbt/util/misc/test_indiferent_hash.rb

@@ -0,0 +1,14 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestIndiferentHash < Test::Unit::TestCase
+  def test_recursive
+    a = {:a => {:b => 1 } }
+    IndiferentHash.setup(a)
+
+    assert IndiferentHash === a["a"]
+    assert_equal 1, a[:a]["b"]
+    assert_equal 1, a["a"][:b]
+  end
+end
+

data/test/rbbt/util/test_cmd.rb

@@ -39,7 +39,7 @@ class TestCmd < Test::Unit::TestCase
     assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :no_fail => true, :pipe => true) end
  
     assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
-    assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
+    assert_raise ConcurrentStreamProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
   end
 
   def test_pipes
@@ -78,4 +78,10 @@ line33
   def test_bash
     puts CMD.bash("awk 'test'")
   end
+
+  def test_cmd_error
+    assert_raise ConcurrentStreamProcessFailed do
+      CMD.cmd_log("ruby -e 'puts 1; STDERR.puts 1; sleep 2; raise'")
+    end
+  end
 end

data/test/rbbt/util/test_open.rb

@@ -98,7 +98,7 @@ class TestOpen < Test::Unit::TestCase
 4
     EOF
     TmpFile.with_file(content) do |file|
-      `bgzip #{file}`
+      `gzip #{file}`
       assert_equal(content, Open.read(file + '.gz'))
       puts content
       FileUtils.rm file + '.gz'
@@ -181,8 +181,8 @@ class TestOpen < Test::Unit::TestCase
       Misc.consume_stream(StringIO.new(text), false, file)
 
       assert_equal text, Open.read(file)
-      assert !File.exist?(file)
-      assert Open.exists? file
+      assert Open.exists?(file)
+      refute File.exist?(file)
     end
 
   end

data/test/rbbt/util/test_ssh.rb

@@ -0,0 +1,10 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+
+    assert TrueClass === SSHLine.rbbt('turbo', 'true')
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.38.0
+  version: 5.38.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-11 00:00:00.000000000 Z
+date: 2023-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -530,6 +530,7 @@ files:
 - test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_format.rb
+- test/rbbt/util/misc/test_indiferent_hash.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
@@ -556,6 +557,7 @@ files:
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/test_ssh.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/step/test_save_load_inputs.rb
@@ -587,7 +589,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.8
+rubygems_version: 3.4.13
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -608,6 +610,7 @@ test_files:
 - test/rbbt/util/test_procpath.rb
 - test/rbbt/util/python/test_util.rb
 - test/rbbt/util/misc/test_development.rb
+- test/rbbt/util/misc/test_indiferent_hash.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_serialize.rb
@@ -630,6 +633,7 @@ test_files:
 - test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_ssh.rb
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/simpleopt/test_setup.rb