rbbt-util 5.28.5 → 5.28.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dc2c06d6d3e5f783f24a671c0150d6cb2da641d003770ee5315c10118a0ea34
-  data.tar.gz: 0e5cf607bd69452be51bf21d36ceeef49ab3d0cb2821728900547e7db60a03af
+  metadata.gz: 443812f3080048a2e39b95ceda4fda2b3f403788c4d32ae4822b0b6920ac1d1f
+  data.tar.gz: e586076cca9dbd72b9dae8ba8be0061269e9bef5d04163f382f2d3b3f353d1b8
 SHA512:
-  metadata.gz: 6d949e77540e01389eb90a757dbf7a499e6aadf237b998b533ffb964fee46bf2ff9c7145c00fa355ce4ff30a1710caab9169bebbf5be22a998c74cc84610c11e
-  data.tar.gz: c2d18ea2d6b45db2878c4a8ae07ac433d1ed365813bf7fe5635c325b743415575998ed6ae3db64d6429719c369ab8be5f263149d9fee8d3ca6edba15b0da1265
+  metadata.gz: ca6ea9a139a48dfcc76fb04b889d45881d023e9f6fda362da0bb7bd4c63b31217ca0a9bb9f4702186c78b741a1eb1b5c7ca2607512e6dced51026081bdcb8aa7
+  data.tar.gz: 76f021c8d565e2833506f17fb54436d0fe80d29d8bface099cece0445cb70718f8f2126e6fa6eb139f6555e33fe9382d1401339aacb100073bb4964c62f1a260

data/lib/rbbt/entity.rb

@@ -38,7 +38,7 @@ module Entity
                                                    if value.to_s == k.to_s
                                                      found = k
                                                      break
-                                                   elsif value =~ /\(#{Regexp.quote k}\)/
+                                                   elsif value.to_s =~ /\(#{Regexp.quote k}\)/
                                                      found = k
                                                      break
                                                    end

data/lib/rbbt/fix_width_table.rb

@@ -67,7 +67,7 @@ class FixWidthTable
 
   def format(pos, value)
     padding = value_size - value.length
-    if range
+    if @range
       (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
     else
       [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
@@ -105,7 +105,7 @@ class FixWidthTable
 
   def idx_value(index)
     return nil if index < 0 or index >= size
-    @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    @file.seek((@range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
     padding = @file.read(4).unpack("l").first+1
     txt = @file.read(value_size)
     str = txt.unpack(mask).first
@@ -277,7 +277,8 @@ class FixWidthTable
 
   def [](pos)
     return [] if size == 0
-    if range
+    self.read
+    if @range
       get_range(pos)
     else
       get_point(pos)
@@ -286,7 +287,7 @@ class FixWidthTable
   
   def overlaps(pos, value = false)
     return [] if size == 0
-    idxs = if range
+    idxs = if @range
       get_range(pos, true)
     else
       get_point(pos, true)

data/lib/rbbt/persist.rb

@@ -25,6 +25,7 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
   def self.newer?(path, file)
     return true if not Open.exists?(file)
     path = path.find if Path === path

data/lib/rbbt/persist/tsv/adapter.rb

@@ -52,7 +52,6 @@ module Persist
     def lock
       return yield if @locked
       lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Log.stack caller if $LOG
       Misc.lock(lock_filename) do
         begin
           @locked = true

data/lib/rbbt/persist/tsv/fix_width_table.rb

@@ -52,7 +52,7 @@ module Persist
       if TSV::ENTRY_KEYS.include? key
         set_metadata(key, value)
       else
-        if range
+        if @range
           add_range_point key, value
         else
           add key, value
@@ -61,7 +61,7 @@ module Persist
     end
      
     def add(key, value)
-      key = pos_function.call(key) if pos_function and not (range and Array === key)
+      key = pos_function.call(key) if pos_function and not (@range and Array === key)
       super(key, value)
     end
 
@@ -85,8 +85,10 @@ module Persist
     end
 
     def each
+      read
       @size.times do |i|
-        yield i, value(i)
+        v = idx_value(i)
+        yield i, v
       end
     end
 

data/lib/rbbt/tsv/accessor.rb

@@ -550,9 +550,17 @@ module TSV
         "\t" << ([""] * fields.length) * "\t" << "\n"
       end
     when Array
-      "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      end
     else
-      "\t" << values.to_s << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.to_s << "\n"
+      end
     end
   end
 

data/lib/rbbt/tsv/dumper.rb

@@ -32,9 +32,21 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        if fields.nil? 
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        elsif fields.empty?
+          "\n"
+        else
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        end
       else
-        sep + values.to_s << "\n"
+        if fields.nil?
+          sep + values.to_s + "\n"
+        elsif fields.empty?
+          "\n"
+        else
+          sep + values.to_s << "\n"
+        end
       end
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -377,6 +377,7 @@ module TSV
         obj.run(true) unless obj.done? || obj.started? || obj.result
 
         stream = obj.get_stream
+        options = {:type => :array}.merge(options) if obj.result_type == :array
 
         if stream
           traverse_obj(stream, options, &block)
@@ -623,6 +624,8 @@ module TSV
   def self.traverse(obj, options = {}, &block)
     into = options[:into]
 
+    into = options[:into] = Open.open(into, :mode => "w") if Misc.is_filename?(into)
+
     case into
     when :stream
       sout = Misc.open_pipe false, false do |sin|                                                                                                                                           

data/lib/rbbt/tsv/util.rb

@@ -184,7 +184,11 @@ module TSV
     str = "" 
     str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
-      str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      if fields.empty?
+        str << header_hash << (key_field || "ID").to_s << "\n" 
+      else
+        str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      end
     end
 
     str

data/lib/rbbt/util/cmd.rb

@@ -98,6 +98,7 @@ module CMD
     post       = options.delete(:post)
     log        = options.delete(:log)
     no_fail    = options.delete(:no_fail)
+    no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
 
     dont_close_in  = options.delete(:dont_close_in)

data/lib/rbbt/util/config.rb

@@ -15,7 +15,7 @@ module Rbbt::Config
   end
 
   def self.load_file(file)
-    Log.debug "Loading file: #{ file }"
+    Log.debug "Loading config file: #{ file }"
     TSV.traverse file, :type => :array do |line|
       next if line =~ /^#/
       key, value, *tokens = line.strip.split(/\s/)
@@ -85,6 +85,7 @@ module Rbbt::Config
     priorities
   end
 
+  # For equal priorities the matching prioritizes tokens ealier in the list
   def self.get(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]

data/lib/rbbt/util/misc/bgzf.rb

@@ -7,7 +7,7 @@ module Bgzf
 
   def self.bgzip_cmd
     @@bgzip_cmd ||= begin
-                      path = `bash -c "type -p bgzips"`.strip
+                      path = `bash -c "type -p bgzip"`.strip
                       if path.empty?
                         Rbbt.claim Rbbt.software.opt.htslib, :install, Rbbt.share.install.software.HTSLIB.find(:lib)
                         Rbbt.software.opt.htslib.produce

data/lib/rbbt/util/misc/inspect.rb

@@ -292,7 +292,7 @@ module Misc
                 files = obj.glob("**/*")
                 "directory: #{Misc.fingerprint(files)}"
               else
-                "file: " << obj << "--" << mtime_str(obj)
+                "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
               end
             else
               obj + " (file missing)"

data/lib/rbbt/util/misc/system.rb

@@ -113,7 +113,7 @@ end
   end
 
   def self.is_filename?(string)
-    return true if defined? PATH and Path === string
+    return true if defined? Path and Path === string
     return true if string.respond_to? :exists
     return true if String === string and string.length < 265 and File.exist?(string)
     return false

data/lib/rbbt/util/named_array.rb

@@ -117,7 +117,7 @@ module NamedArray
   #end
 
   def each(&block)
-    if defined?(Entity) and not @fields.nil? and not @fields.empty?
+    if defined?(Entity) && ! (@fields.nil? || @fields.empty?)
       i = 0
       super do |elem|
         field = @fields[i]

data/lib/rbbt/util/open.rb

@@ -16,9 +16,9 @@ module Open
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
                  ENV["GREP_CMD"]
-               elsif File.exists?('/bin/grep')
+               elsif File.exist?('/bin/grep')
                  "/bin/grep"
-               elsif File.exists?('/usr/bin/grep')
+               elsif File.exist?('/usr/bin/grep')
                  "/usr/bin/grep"
                else
                  "grep"
@@ -262,7 +262,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       remove_from_repo(*dir_sub_path)
     else
-      FileUtils.rm(file) if File.exists?(file) or Open.broken_link?(file)
+      FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
     end
   end
 
@@ -333,7 +333,7 @@ module Open
       nil
     else
       target = target.find if Path === target
-      if ! File.exists?(target)
+      if ! File.exist?(target)
         FileUtils.mkdir_p target
       end
     end
@@ -344,8 +344,8 @@ module Open
     target = target.find if Path === target
 
     target = File.join(target, File.basename(source)) if File.directory? target
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln_s source, target
   end
 
@@ -353,8 +353,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln source, target
   end
 
@@ -362,8 +362,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     begin
       CMD.cmd("ln -L '#{ source }' '#{ target }'")
     rescue ProcessFailed
@@ -397,7 +397,7 @@ module Open
     if dir_sub_path_source.nil? and dir_sub_path_target.nil?
       FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
       tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
-      FileUtils.cp source, tmp_target
+      FileUtils.cp_r source, tmp_target
       FileUtils.mv tmp_target, target
       return
     end
@@ -475,6 +475,7 @@ module Open
       File.exist?(file) #|| File.symlink?(file)
     end
   end
+
   class << self
     alias exist? exists?
   end
@@ -747,7 +748,7 @@ module Open
     if (dir_sub_path = find_repo_dir(path))
       writable_repo?(*dir_sub_path)
     else
-      if File.exists?(path)
+      if File.exist?(path)
         File.writable?(path)
       else
         File.writable?(File.dirname(File.expand_path(path)))
@@ -776,14 +777,14 @@ module Open
       file = file.find if Path === file
       begin
         if File.symlink?(file) || File.stat(file).nlink > 1
-          if File.exists?(file + '.info') && defined?(Step)
+          if File.exist?(file + '.info') && defined?(Step)
             done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
             return done if done
           end
 
           file = Pathname.new(file).realpath.to_s 
         end
-        return nil unless File.exists?(file)
+        return nil unless File.exist?(file)
         File.mtime(file)
       rescue
         nil
@@ -793,7 +794,7 @@ module Open
 
   def self.update_mtime(path, target)
     if File.symlink?(target) || File.stat(target).nlink > 1
-      if File.exists?(target + '.info')
+      if File.exist?(target + '.info')
         target = target + '.info'
       else
         target = Pathname.new(target).realpath.to_s 
@@ -801,7 +802,7 @@ module Open
     end
 
     CMD.cmd("touch -r '#{path}' '#{target}'")
-    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exists?(path + '.info')
+    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exist?(path + '.info')
   end
 
   def self.atime(file)
@@ -823,7 +824,7 @@ module Open
   end
 
   def self.broken_link?(path)
-    File.symlink?(path) && ! File.exists?(File.readlink(path))
+    File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
   def self.download(url, path)

data/lib/rbbt/workflow.rb

@@ -364,6 +364,7 @@ module Workflow
       raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
     end
 
+    # jobname => true sets the value of the input to the name of the job
     if task.input_options
       jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
     else

data/lib/rbbt/workflow/accessor.rb

@@ -26,105 +26,106 @@ module Workflow
     raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
     id = File.join(self.to_s, name.to_s)
     @task_info ||= {}
-    @task_info[id] ||= begin description = task.description
-                           result_description = task.result_description
-                           result_type = task.result_type
-                           inputs = rec_inputs(name).uniq
-                           input_types = rec_input_types(name)
-                           input_descriptions = rec_input_descriptions(name)
-                           input_use = rec_input_use(name)
-                           input_defaults = rec_input_defaults(name)
-                           input_options = rec_input_options(name)
-                           extension = task.extension
-                           export = case
-                                    when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
-                                      :synchronous
-                                    when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
-                                      :asynchronous
-                                    when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
-                                      :exec
-                                    when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
-                                      :stream
-                                    else
-                                      :none
-                                    end
+    @task_info[id] ||= begin 
+                         description = task.description
+                         result_description = task.result_description
+                         result_type = task.result_type
+                         inputs = rec_inputs(name).uniq
+                         input_types = rec_input_types(name)
+                         input_descriptions = rec_input_descriptions(name)
+                         input_use = rec_input_use(name)
+                         input_defaults = rec_input_defaults(name)
+                         input_options = rec_input_options(name)
+                         extension = task.extension
+                         export = case
+                                  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
+                                    :synchronous
+                                  when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
+                                    :asynchronous
+                                  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
+                                    :exec
+                                  when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
+                                    :stream
+                                  else
+                                    :none
+                                  end
 
-                           dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
-                           info = { :id => id,
-                             :description => description,
-                             :export => export,
-                             :inputs => inputs,
-                             :input_types => input_types,
-                             :input_descriptions => input_descriptions,
-                             :input_defaults => input_defaults,
-                             :input_options => input_options,
-                             :input_use => input_use,
-                             :result_type => result_type,
-                             :result_description => result_description,
-                             :dependencies => dependencies,
-                             :extension => extension
-                           }
-                         end
+                         dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
+                         { :id => id,
+                           :description => description,
+                           :export => export,
+                           :inputs => inputs,
+                           :input_types => input_types,
+                           :input_descriptions => input_descriptions,
+                           :input_defaults => input_defaults,
+                           :input_options => input_options,
+                           :input_use => input_use,
+                           :result_type => result_type,
+                           :result_description => result_description,
+                           :dependencies => dependencies,
+                           :extension => extension
+                         }
+                       end
   end
 
   def rec_dependencies(taskname)
     @rec_dependencies ||= {}
     @rec_dependencies[taskname] ||= begin
-                            if task_dependencies.include? taskname
-
-                              deps = task_dependencies[taskname]
-
-                              #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
-
-                              all_deps = []
-                              deps.each do |dep| 
-                                if DependencyBlock === dep
-                                  all_deps << dep.dependency if dep.dependency
-                                else
-                                  all_deps << dep unless Proc === dep
-                                end
-
-                                begin
-                                  case dep
-                                  when Array
-                                    wf, t, o = dep
-
-                                    wf.rec_dependencies(t.to_sym).each do |d|
-                                      if Array === d
-                                        new = d.dup
-                                      else
-                                        new = [dep.first, d]
-                                      end
-
-                                      if Hash === o and not o.empty? 
-                                        if Hash === new.last
-                                          hash = new.last.dup
-                                          o.each{|k,v| hash[k] ||= v}
-                                          new[new.length-1] = hash
-                                        else
-                                          new.push o.dup
+                                      if task_dependencies.include? taskname
+
+                                        deps = task_dependencies[taskname]
+
+                                        #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
+
+                                        all_deps = []
+                                        deps.each do |dep| 
+                                          if DependencyBlock === dep
+                                            all_deps << dep.dependency if dep.dependency
+                                          else
+                                            all_deps << dep unless Proc === dep
+                                          end
+
+                                          begin
+                                            case dep
+                                            when Array
+                                              wf, t, o = dep
+
+                                              wf.rec_dependencies(t.to_sym).each do |d|
+                                                if Array === d
+                                                  new = d.dup
+                                                else
+                                                  new = [dep.first, d]
+                                                end
+
+                                                if Hash === o and not o.empty? 
+                                                  if Hash === new.last
+                                                    hash = new.last.dup
+                                                    o.each{|k,v| hash[k] ||= v}
+                                                    new[new.length-1] = hash
+                                                  else
+                                                    new.push o.dup
+                                                  end
+                                                end
+
+                                                all_deps << new
+                                              end if wf && t
+
+                                            when String, Symbol
+                                              rec_deps = rec_dependencies(dep.to_sym)
+                                              all_deps.concat rec_deps
+                                            when DependencyBlock
+                                              dep = dep.dependency
+                                              raise TryAgain
+                                            end
+                                          rescue TryAgain
+                                            retry
+                                          end
                                         end
+                                        all_deps.uniq
+                                      else
+                                        []
                                       end
-
-                                      all_deps << new
-                                    end if wf && t
-
-                                  when String, Symbol
-                                    rec_deps = rec_dependencies(dep.to_sym)
-                                    all_deps.concat rec_deps
-                                  when DependencyBlock
-                                    dep = dep.dependency
-                                    raise TryAgain
-                                  end
-                                rescue TryAgain
-                                  retry
-                                end
-                              end
-                              all_deps.uniq
-                            else
-                              []
-                            end
-                          end
+                                    end
   end
 
   def task_from_dep(dep)
@@ -182,7 +183,7 @@ module Workflow
       acc
     }.tap{|h| IndiferentHash.setup(h)}
   end
-  
+
   def rec_input_use(taskname)
     task = task_from_dep(taskname)
     deps = rec_dependencies(taskname)
@@ -474,7 +475,7 @@ module Workflow
     else
       workdir_find = workdir
     end
- 
+
     workdir_find = File.expand_path(workdir_find)
     path = File.expand_path(path)
     dir = File.dirname(path)