rbbt-util 5.28.1 → 5.28.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49e2972467a5efda29a81c562f96a69f190415a9226c4e8e9e5e776bc1e44f33
-  data.tar.gz: 047125157bbeb36260736630c091dda1243a7856f88b01553c6ecec5177c1795
+  metadata.gz: f13812c850239f7d78c94e74288ea7807663c72146002e13a13aa99babb18555
+  data.tar.gz: 3568f646d3095448ac6c8d99f32a5513421c80fd9d8b1f9a049a7ce512f3139d
 SHA512:
-  metadata.gz: ddd94924f8cdbbe8b9219d64a8261e484e0f7c1c34ee6b4b6a745e9dca2e9f51524f6d1d45471cd2ce229661500fa8052515ce5835fb0061f1ef07021b5eb37f
-  data.tar.gz: c5f52161994f7d6de86aba05544f05dfbe1fff765c587220adca9a917c6f7ebe410cfeae601996eaa790f9587468508a8750618a5c126b1ad5e07238bc4e34e1
+  metadata.gz: 644bcc769e818310492993b15a024b83de399d65a0cfc74b9dc06ee233bfd66da3c0951a26899c66ce62a8410104f00b8b84c07f77a4a1159a7fb5149befb5a1
+  data.tar.gz: da5fd395014feded588f223686bd032dd86e46cac0e48e4369e9d1ea2e1f0d021e0fab02539ab2be1f483efc5463b9487952783edf2e00f22d7158879b5fd32d

data/lib/rbbt/hpc.rb

@@ -451,12 +451,15 @@ EOF
           out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
           err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
 
-          Misc.consume_stream(err, true, STDERR) if err
-          Misc.consume_stream(out, true, STDOUT) if out
+          terr = Misc.consume_stream(err, true, STDERR) if err
+          tout = Misc.consume_stream(out, true, STDOUT) if out
 
           sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
+        rescue Aborted
         ensure
           begin
+            terr.exit if terr
+            tout.exit if tout
             err.close if err
             err.join if err
           rescue Exception

data/lib/rbbt/persist.rb

@@ -25,6 +25,7 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
+  # Return non-false if the first file is newer than the second file
   def self.newer?(path, file)
     return true if not Open.exists?(file)
     path = path.find if Path === path

data/lib/rbbt/resource/path.rb

@@ -412,7 +412,9 @@ module Path
   end
 
   def replace_extension(new_extension = nil, multiple = false)
-    if multiple
+    if String === multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
+    elsif multiple
       new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
     else
       new_path = self.sub(/\.[^\.\/]{1,5}$/,'')

data/lib/rbbt/tsv/accessor.rb

@@ -550,9 +550,17 @@ module TSV
         "\t" << ([""] * fields.length) * "\t" << "\n"
       end
     when Array
-      "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      end
     else
-      "\t" << values.to_s << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.to_s << "\n"
+      end
     end
   end
 

data/lib/rbbt/tsv/change_id.rb

@@ -58,13 +58,13 @@ module TSV
 
     identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
     identifiers = tsv.identifier_files.first if identifiers.nil?
-    identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? and tsv.namespace and Organism.identifiers(tsv.namespace).exists?
+    identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
     identifiers.namespace ||= tsv.namespace
 
     fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil 
     #index = identifiers.index :target => format, :fields => fields, :persist => persist_input, :order => true
 
-    grep = Organism.blacklist_genes(tsv.namespace).list  if identifiers.namespace and Organism.blacklist_genes(tsv.namespace).exists?
+    grep = Organism.blacklist_genes(tsv.namespace).list  if defined?(Organism) && identifiers.namespace && Organism.blacklist_genes(tsv.namespace).exists?
     if fields.nil?
       index = identifiers.index(:data_tsv_grep => grep, :data_invert_grep => true, :target => format, :persist => true, :order => true, :unnamed => true, :data_persist => true)
     else

data/lib/rbbt/tsv/dumper.rb

@@ -32,9 +32,17 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        end
       else
-        sep + values.to_s << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + values.to_s << "\n"
+        end
       end
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -101,8 +101,8 @@ module TSV
         error = true
         raise $!
       ensure
-        Log::ProgressBar.remove_bar(bar) if bar
         join.call(error) if join
+        Log::ProgressBar.remove_bar(bar) if bar
       end
   end
 
@@ -137,8 +137,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -177,8 +177,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -219,8 +219,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -273,8 +273,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -318,8 +318,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -377,6 +377,7 @@ module TSV
         obj.run(true) unless obj.done? || obj.started? || obj.result
 
         stream = obj.get_stream
+        options = {:type => :array}.merge(options) if obj.result_type == :array
 
         if stream
           traverse_obj(stream, options, &block)
@@ -623,6 +624,8 @@ module TSV
   def self.traverse(obj, options = {}, &block)
     into = options[:into]
 
+    into = options[:into] = Open.open(into, :mode => "w") if Misc.is_filename?(into)
+
     case into
     when :stream
       sout = Misc.open_pipe false, false do |sin|                                                                                                                                           
@@ -727,8 +730,6 @@ module TSV
         Log.exception $!
         abort_stream(into, $!)
         raise $!
-      ensure
-        Log::ProgressBar.remove_bar(bar) if bar
       end
 
       into

data/lib/rbbt/tsv/util.rb

@@ -184,7 +184,11 @@ module TSV
     str = "" 
     str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
-      str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      if fields.empty?
+        str << header_hash << (key_field || "ID").to_s << "\n" 
+      else
+        str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      end
     end
 
     str
@@ -316,22 +320,30 @@ module TSV
 
   def to_single
     new = {}
-    case type
-    when :double
-      through do |k,v|
-        new[k] = v.first.first
-      end
-    when :flat
+
+    if block_given?
       through do |k,v|
-        new[k] = v.first
+        new[k] = yield v
       end
-    when :single
-      return self
-    when :list
-      through do |k,v|
-        new[k] = v.first
+    else
+      case type
+      when :double
+        through do |k,v|
+          new[k] = v.first.first
+        end
+      when :flat
+        through do |k,v|
+          new[k] = v.first
+        end
+      when :single
+        return self
+      when :list
+        through do |k,v|
+          new[k] = v.first
+        end
       end
     end
+
     self.annotate(new)
     new.type = :single
     new.fields = [new.fields.first] if new.fields.length > 1

data/lib/rbbt/util/config.rb

@@ -15,7 +15,7 @@ module Rbbt::Config
   end
 
   def self.load_file(file)
-    Log.debug "Loading file: #{ file }"
+    Log.debug "Loading config file: #{ file }"
     TSV.traverse file, :type => :array do |line|
       next if line =~ /^#/
       key, value, *tokens = line.strip.split(/\s/)
@@ -85,6 +85,7 @@ module Rbbt::Config
     priorities
   end
 
+  # For equal priorities the matching prioritizes tokens ealier in the list
   def self.get(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]

data/lib/rbbt/util/misc/inspect.rb

@@ -292,7 +292,7 @@ module Misc
                 files = obj.glob("**/*")
                 "directory: #{Misc.fingerprint(files)}"
               else
-                "file: " << obj << "--" << mtime_str(obj)
+                "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
               end
             else
               obj + " (file missing)"
@@ -401,7 +401,7 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
     return nil if m.nil?
     m[1]
   end

data/lib/rbbt/util/misc/system.rb

@@ -113,7 +113,7 @@ end
   end
 
   def self.is_filename?(string)
-    return true if defined? PATH and Path === string
+    return true if defined? Path and Path === string
     return true if string.respond_to? :exists
     return true if String === string and string.length < 265 and File.exist?(string)
     return false

data/lib/rbbt/workflow.rb

@@ -364,6 +364,7 @@ module Workflow
       raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
     end
 
+    # jobname => true sets the value of the input to the name of the job
     if task.input_options
       jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
     else

data/lib/rbbt/workflow/accessor.rb

@@ -26,105 +26,106 @@ module Workflow
     raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
     id = File.join(self.to_s, name.to_s)
     @task_info ||= {}
-    @task_info[id] ||= begin description = task.description
-                           result_description = task.result_description
-                           result_type = task.result_type
-                           inputs = rec_inputs(name).uniq
-                           input_types = rec_input_types(name)
-                           input_descriptions = rec_input_descriptions(name)
-                           input_use = rec_input_use(name)
-                           input_defaults = rec_input_defaults(name)
-                           input_options = rec_input_options(name)
-                           extension = task.extension
-                           export = case
-                                    when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
-                                      :synchronous
-                                    when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
-                                      :asynchronous
-                                    when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
-                                      :exec
-                                    when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
-                                      :stream
-                                    else
-                                      :none
-                                    end
+    @task_info[id] ||= begin 
+                         description = task.description
+                         result_description = task.result_description
+                         result_type = task.result_type
+                         inputs = rec_inputs(name).uniq
+                         input_types = rec_input_types(name)
+                         input_descriptions = rec_input_descriptions(name)
+                         input_use = rec_input_use(name)
+                         input_defaults = rec_input_defaults(name)
+                         input_options = rec_input_options(name)
+                         extension = task.extension
+                         export = case
+                                  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
+                                    :synchronous
+                                  when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
+                                    :asynchronous
+                                  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
+                                    :exec
+                                  when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
+                                    :stream
+                                  else
+                                    :none
+                                  end
 
-                           dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
-                           info = { :id => id,
-                             :description => description,
-                             :export => export,
-                             :inputs => inputs,
-                             :input_types => input_types,
-                             :input_descriptions => input_descriptions,
-                             :input_defaults => input_defaults,
-                             :input_options => input_options,
-                             :input_use => input_use,
-                             :result_type => result_type,
-                             :result_description => result_description,
-                             :dependencies => dependencies,
-                             :extension => extension
-                           }
-                         end
+                         dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
+                         { :id => id,
+                           :description => description,
+                           :export => export,
+                           :inputs => inputs,
+                           :input_types => input_types,
+                           :input_descriptions => input_descriptions,
+                           :input_defaults => input_defaults,
+                           :input_options => input_options,
+                           :input_use => input_use,
+                           :result_type => result_type,
+                           :result_description => result_description,
+                           :dependencies => dependencies,
+                           :extension => extension
+                         }
+                       end
   end
 
   def rec_dependencies(taskname)
     @rec_dependencies ||= {}
     @rec_dependencies[taskname] ||= begin
-                            if task_dependencies.include? taskname
-
-                              deps = task_dependencies[taskname]
-
-                              #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
-
-                              all_deps = []
-                              deps.each do |dep| 
-                                if DependencyBlock === dep
-                                  all_deps << dep.dependency if dep.dependency
-                                else
-                                  all_deps << dep unless Proc === dep
-                                end
-
-                                begin
-                                  case dep
-                                  when Array
-                                    wf, t, o = dep
-
-                                    wf.rec_dependencies(t.to_sym).each do |d|
-                                      if Array === d
-                                        new = d.dup
-                                      else
-                                        new = [dep.first, d]
-                                      end
-
-                                      if Hash === o and not o.empty? 
-                                        if Hash === new.last
-                                          hash = new.last.dup
-                                          o.each{|k,v| hash[k] ||= v}
-                                          new[new.length-1] = hash
-                                        else
-                                          new.push o.dup
+                                      if task_dependencies.include? taskname
+
+                                        deps = task_dependencies[taskname]
+
+                                        #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
+
+                                        all_deps = []
+                                        deps.each do |dep| 
+                                          if DependencyBlock === dep
+                                            all_deps << dep.dependency if dep.dependency
+                                          else
+                                            all_deps << dep unless Proc === dep
+                                          end
+
+                                          begin
+                                            case dep
+                                            when Array
+                                              wf, t, o = dep
+
+                                              wf.rec_dependencies(t.to_sym).each do |d|
+                                                if Array === d
+                                                  new = d.dup
+                                                else
+                                                  new = [dep.first, d]
+                                                end
+
+                                                if Hash === o and not o.empty? 
+                                                  if Hash === new.last
+                                                    hash = new.last.dup
+                                                    o.each{|k,v| hash[k] ||= v}
+                                                    new[new.length-1] = hash
+                                                  else
+                                                    new.push o.dup
+                                                  end
+                                                end
+
+                                                all_deps << new
+                                              end if wf && t
+
+                                            when String, Symbol
+                                              rec_deps = rec_dependencies(dep.to_sym)
+                                              all_deps.concat rec_deps
+                                            when DependencyBlock
+                                              dep = dep.dependency
+                                              raise TryAgain
+                                            end
+                                          rescue TryAgain
+                                            retry
+                                          end
                                         end
+                                        all_deps.uniq
+                                      else
+                                        []
                                       end
-
-                                      all_deps << new
-                                    end if wf && t
-
-                                  when String, Symbol
-                                    rec_deps = rec_dependencies(dep.to_sym)
-                                    all_deps.concat rec_deps
-                                  when DependencyBlock
-                                    dep = dep.dependency
-                                    raise TryAgain
-                                  end
-                                rescue TryAgain
-                                  retry
-                                end
-                              end
-                              all_deps.uniq
-                            else
-                              []
-                            end
-                          end
+                                    end
   end
 
   def task_from_dep(dep)
@@ -182,7 +183,7 @@ module Workflow
       acc
     }.tap{|h| IndiferentHash.setup(h)}
   end
-  
+
   def rec_input_use(taskname)
     task = task_from_dep(taskname)
     deps = rec_dependencies(taskname)
@@ -474,7 +475,7 @@ module Workflow
     else
       workdir_find = workdir
     end
- 
+
     workdir_find = File.expand_path(workdir_find)
     path = File.expand_path(path)
     dir = File.dirname(path)