rbbt-util 5.28.4 → 5.28.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eb8b404c84b1c6d07f3d13ffeaeb97a4b9fd59bf237241b98e184cbfdb98a43
-  data.tar.gz: f2a0e7c55aa053a7d559143b1ae235558355fd1f4f61f2845864e47b621e37fc
+  metadata.gz: 2fa2f71df9aef8810599ab3660c847c084dea75ccb2485d294cfa1962e341162
+  data.tar.gz: 7eb004ddf8bcf30b00325dc5251d9ed73a307de433c18c0847eb4d70a4bea1e2
 SHA512:
-  metadata.gz: fd8348766af38ef48c379966ae2766e59fcb14714d78a5130856cdfef2d5a87eeabb4ae519a2fa6a2fa862d6b71cfff711eefe293b3f0954af6b0f70bbf58bdd
-  data.tar.gz: a8187db527df84d7db46e1cc049ed8e3a850188da16d0834408af102bf7f02977524786ab1267e6d047744d3e8cca3cc02c1382ede7b8766a61468867e2f8464
+  metadata.gz: 6ba23322f9ac768f1782c4e26605c50171b6d35d1d3c2acc0ef195e26ca39af851a232e1581ca2ec220899b7780f35f8d0d36e1f61f0b0e6500c883999c3760b
+  data.tar.gz: afa0952f228a951ea78506ad0776fd96dbfbd2b91bac73310ee6807e4a7d0a74c7ca8f582880c5abd28bbf709c8be3d9dff1797056be756962091b2e97031f30

data/lib/rbbt/hpc.rb

@@ -451,12 +451,15 @@ EOF
           out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
           err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
 
-          Misc.consume_stream(err, true, STDERR) if err
-          Misc.consume_stream(out, true, STDOUT) if out
+          terr = Misc.consume_stream(err, true, STDERR) if err
+          tout = Misc.consume_stream(out, true, STDOUT) if out
 
           sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
+        rescue Aborted
         ensure
           begin
+            terr.exit if terr
+            tout.exit if tout
             err.close if err
             err.join if err
           rescue Exception

data/lib/rbbt/persist.rb

@@ -25,6 +25,7 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
+  # Return non-false if the first file is newer than the second file
   def self.newer?(path, file)
     return true if not Open.exists?(file)
     path = path.find if Path === path

data/lib/rbbt/tsv/accessor.rb

@@ -550,9 +550,17 @@ module TSV
         "\t" << ([""] * fields.length) * "\t" << "\n"
       end
     when Array
-      "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      end
     else
-      "\t" << values.to_s << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.to_s << "\n"
+      end
     end
   end
 

data/lib/rbbt/tsv/dumper.rb

@@ -32,9 +32,17 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        end
       else
-        sep + values.to_s << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + values.to_s << "\n"
+        end
       end
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -377,6 +377,7 @@ module TSV
         obj.run(true) unless obj.done? || obj.started? || obj.result
 
         stream = obj.get_stream
+        options = {:type => :array}.merge(options) if obj.result_type == :array
 
         if stream
           traverse_obj(stream, options, &block)
@@ -623,6 +624,8 @@ module TSV
   def self.traverse(obj, options = {}, &block)
     into = options[:into]
 
+    into = options[:into] = Open.open(into, :mode => "w") if Misc.is_filename?(into)
+
     case into
     when :stream
       sout = Misc.open_pipe false, false do |sin|                                                                                                                                           

data/lib/rbbt/tsv/util.rb

@@ -184,7 +184,11 @@ module TSV
     str = "" 
     str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
-      str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      if fields.empty?
+        str << header_hash << (key_field || "ID").to_s << "\n" 
+      else
+        str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      end
     end
 
     str

data/lib/rbbt/util/config.rb

@@ -15,7 +15,7 @@ module Rbbt::Config
   end
 
   def self.load_file(file)
-    Log.debug "Loading file: #{ file }"
+    Log.debug "Loading config file: #{ file }"
     TSV.traverse file, :type => :array do |line|
       next if line =~ /^#/
       key, value, *tokens = line.strip.split(/\s/)
@@ -85,6 +85,7 @@ module Rbbt::Config
     priorities
   end
 
+  # For equal priorities the matching prioritizes tokens ealier in the list
   def self.get(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]

data/lib/rbbt/util/misc/inspect.rb

@@ -292,7 +292,7 @@ module Misc
                 files = obj.glob("**/*")
                 "directory: #{Misc.fingerprint(files)}"
               else
-                "file: " << obj << "--" << mtime_str(obj)
+                "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
               end
             else
               obj + " (file missing)"

data/lib/rbbt/util/misc/system.rb

@@ -113,7 +113,7 @@ end
   end
 
   def self.is_filename?(string)
-    return true if defined? PATH and Path === string
+    return true if defined? Path and Path === string
     return true if string.respond_to? :exists
     return true if String === string and string.length < 265 and File.exist?(string)
     return false

data/lib/rbbt/util/open.rb

@@ -397,7 +397,7 @@ module Open
     if dir_sub_path_source.nil? and dir_sub_path_target.nil?
       FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
       tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
-      FileUtils.cp source, tmp_target
+      FileUtils.cp_r source, tmp_target
       FileUtils.mv tmp_target, target
       return
     end

data/lib/rbbt/workflow.rb

@@ -364,6 +364,7 @@ module Workflow
       raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
     end
 
+    # jobname => true sets the value of the input to the name of the job
     if task.input_options
       jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
     else

data/lib/rbbt/workflow/accessor.rb

@@ -26,105 +26,106 @@ module Workflow
     raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
     id = File.join(self.to_s, name.to_s)
     @task_info ||= {}
-    @task_info[id] ||= begin description = task.description
-                           result_description = task.result_description
-                           result_type = task.result_type
-                           inputs = rec_inputs(name).uniq
-                           input_types = rec_input_types(name)
-                           input_descriptions = rec_input_descriptions(name)
-                           input_use = rec_input_use(name)
-                           input_defaults = rec_input_defaults(name)
-                           input_options = rec_input_options(name)
-                           extension = task.extension
-                           export = case
-                                    when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
-                                      :synchronous
-                                    when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
-                                      :asynchronous
-                                    when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
-                                      :exec
-                                    when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
-                                      :stream
-                                    else
-                                      :none
-                                    end
+    @task_info[id] ||= begin 
+                         description = task.description
+                         result_description = task.result_description
+                         result_type = task.result_type
+                         inputs = rec_inputs(name).uniq
+                         input_types = rec_input_types(name)
+                         input_descriptions = rec_input_descriptions(name)
+                         input_use = rec_input_use(name)
+                         input_defaults = rec_input_defaults(name)
+                         input_options = rec_input_options(name)
+                         extension = task.extension
+                         export = case
+                                  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
+                                    :synchronous
+                                  when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
+                                    :asynchronous
+                                  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
+                                    :exec
+                                  when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
+                                    :stream
+                                  else
+                                    :none
+                                  end
 
-                           dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
-                           info = { :id => id,
-                             :description => description,
-                             :export => export,
-                             :inputs => inputs,
-                             :input_types => input_types,
-                             :input_descriptions => input_descriptions,
-                             :input_defaults => input_defaults,
-                             :input_options => input_options,
-                             :input_use => input_use,
-                             :result_type => result_type,
-                             :result_description => result_description,
-                             :dependencies => dependencies,
-                             :extension => extension
-                           }
-                         end
+                         dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
+                         { :id => id,
+                           :description => description,
+                           :export => export,
+                           :inputs => inputs,
+                           :input_types => input_types,
+                           :input_descriptions => input_descriptions,
+                           :input_defaults => input_defaults,
+                           :input_options => input_options,
+                           :input_use => input_use,
+                           :result_type => result_type,
+                           :result_description => result_description,
+                           :dependencies => dependencies,
+                           :extension => extension
+                         }
+                       end
   end
 
   def rec_dependencies(taskname)
     @rec_dependencies ||= {}
     @rec_dependencies[taskname] ||= begin
-                            if task_dependencies.include? taskname
-
-                              deps = task_dependencies[taskname]
-
-                              #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
-
-                              all_deps = []
-                              deps.each do |dep| 
-                                if DependencyBlock === dep
-                                  all_deps << dep.dependency if dep.dependency
-                                else
-                                  all_deps << dep unless Proc === dep
-                                end
-
-                                begin
-                                  case dep
-                                  when Array
-                                    wf, t, o = dep
-
-                                    wf.rec_dependencies(t.to_sym).each do |d|
-                                      if Array === d
-                                        new = d.dup
-                                      else
-                                        new = [dep.first, d]
-                                      end
-
-                                      if Hash === o and not o.empty? 
-                                        if Hash === new.last
-                                          hash = new.last.dup
-                                          o.each{|k,v| hash[k] ||= v}
-                                          new[new.length-1] = hash
-                                        else
-                                          new.push o.dup
+                                      if task_dependencies.include? taskname
+
+                                        deps = task_dependencies[taskname]
+
+                                        #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
+
+                                        all_deps = []
+                                        deps.each do |dep| 
+                                          if DependencyBlock === dep
+                                            all_deps << dep.dependency if dep.dependency
+                                          else
+                                            all_deps << dep unless Proc === dep
+                                          end
+
+                                          begin
+                                            case dep
+                                            when Array
+                                              wf, t, o = dep
+
+                                              wf.rec_dependencies(t.to_sym).each do |d|
+                                                if Array === d
+                                                  new = d.dup
+                                                else
+                                                  new = [dep.first, d]
+                                                end
+
+                                                if Hash === o and not o.empty? 
+                                                  if Hash === new.last
+                                                    hash = new.last.dup
+                                                    o.each{|k,v| hash[k] ||= v}
+                                                    new[new.length-1] = hash
+                                                  else
+                                                    new.push o.dup
+                                                  end
+                                                end
+
+                                                all_deps << new
+                                              end if wf && t
+
+                                            when String, Symbol
+                                              rec_deps = rec_dependencies(dep.to_sym)
+                                              all_deps.concat rec_deps
+                                            when DependencyBlock
+                                              dep = dep.dependency
+                                              raise TryAgain
+                                            end
+                                          rescue TryAgain
+                                            retry
+                                          end
                                         end
+                                        all_deps.uniq
+                                      else
+                                        []
                                       end
-
-                                      all_deps << new
-                                    end if wf && t
-
-                                  when String, Symbol
-                                    rec_deps = rec_dependencies(dep.to_sym)
-                                    all_deps.concat rec_deps
-                                  when DependencyBlock
-                                    dep = dep.dependency
-                                    raise TryAgain
-                                  end
-                                rescue TryAgain
-                                  retry
-                                end
-                              end
-                              all_deps.uniq
-                            else
-                              []
-                            end
-                          end
+                                    end
   end
 
   def task_from_dep(dep)
@@ -182,7 +183,7 @@ module Workflow
       acc
     }.tap{|h| IndiferentHash.setup(h)}
   end
-  
+
   def rec_input_use(taskname)
     task = task_from_dep(taskname)
     deps = rec_dependencies(taskname)
@@ -474,7 +475,7 @@ module Workflow
     else
       workdir_find = workdir
     end
- 
+
     workdir_find = File.expand_path(workdir_find)
     path = File.expand_path(path)
     dir = File.dirname(path)

data/lib/rbbt/workflow/definition.rb

@@ -81,9 +81,10 @@ module Workflow
       if forget
         remove = config :remove_dep_tasks, :remove_dep_tasks, :default => REMOVE_DEP_TASKS
         self.archive_deps
+        self.copy_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir
+        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
         case remove.to_s
         when 'true'
@@ -92,8 +93,10 @@ module Workflow
           dep.recursive_clean
         end
       else
-        Open.rm_rf self.files_dir 
-        Open.link dep.files_dir, self.files_dir
+        if Open.exists?(dep.files_dir)
+          Open.rm_rf self.files_dir 
+          Open.link dep.files_dir, self.files_dir
+        end
         Open.link dep.path, self.path
       end
       nil

data/lib/rbbt/workflow/integration/cromwell.rb

@@ -1,9 +1,9 @@
-module Workflow
+module Cromwell
 
   Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
   Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
 
-  def run_cromwell(file, work_dir, options = {})
+  def self.run_cromwell(file, work_dir, options = {})
     cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
     jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
     if cromwell_inputs_file
@@ -12,7 +12,9 @@ module Workflow
       CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
     end
   end
+end
 
+module Workflow
   def load_cromwell(file)
     jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
     inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -85,7 +85,11 @@ class RemoteWorkflow
 
       RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
       name = RemoteWorkflow.capture_exception do
-        RestClient.post(self.encode(url), params)
+        begin
+          RestClient.post(self.encode(url), params)
+        rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
+          raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
+        end
       end
 
       Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }

data/lib/rbbt/workflow/step.rb

@@ -84,6 +84,7 @@ class Step
   end
 
   def load_dependencies_from_info
+    relocated = nil
     @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
       if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
         Workflow._load_step dep_path
@@ -108,6 +109,14 @@ class Step
     @inputs || []
   end
 
+  def copy_files_dir
+    if File.symlink?(self.files_dir)
+      realpath = Open.realpath(self.files_dir)
+      Open.rm self.files_dir
+      Open.cp realpath, self.files_dir
+    end
+  end
+
   def archive_deps
     self.set_info :archived_info, archived_info
     self.set_info :archived_dependencies, info[:dependencies]
@@ -411,7 +420,7 @@ class Step
       return 
     end
 
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
 
       @result = nil
       @pid = nil
@@ -419,8 +428,8 @@ class Step
       Misc.insist do
         Open.rm info_file if Open.exists?(info_file)
         Open.rm md5_file if Open.exists?(md5_file)
-        Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
-        Open.rm_rf files_dir if Open.exists?(files_dir)
+        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
+        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
         Open.rm pid_file if Open.exists?(pid_file)
         Open.rm tmp_path if Open.exists?(tmp_path)
       end
@@ -454,12 +463,15 @@ class Step
     return [] if dependencies.nil? or dependencies.empty?
 
     new_dependencies = []
+    archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+
     dependencies.each{|step| 
       #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if seen.include? step
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
       next if self.done? && need_run && ! updatable?
 
-      r = step.rec_dependencies(need_run, new_dependencies)
+      r = step.rec_dependencies(need_run, new_dependencies.collect{|d| d.path})
       new_dependencies.concat r
       new_dependencies << step
     }