rbbt-util 5.28.8 → 5.28.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d77757da0912cd41138ff33b99e19b7ffa1449be515fb7a8742f9d69c134ffd
-  data.tar.gz: 1edb139f92881591bbac18692dc6a0a739be9dfe0a820a4db2cffd30079fc328
+  metadata.gz: 74253d97dc17c890ce9022e38ccd514847e785b69f4cfd74a8a5333a9aa1c97d
+  data.tar.gz: d908fecdec1c5e6ccce81b3190883011d775f38cab689c136e7c90808c3c93ac
 SHA512:
-  metadata.gz: 2219bd7152eda55f2c3c9b1e5d10d9593dd29232e7e77fe0707496031030455663983256930f35c2bcabd8b80e15c05d99f3bcdc9644135ee4a75d0267334c33
-  data.tar.gz: e4021aab6574fc10e960eec29f630075c8e95b27d6648ba47e1f76ecbf1a12b84a5859ad7b61ffde8591466ddcfec8ee5cfd27f6f82997648d712225ac099e1b
+  metadata.gz: 7c6b8f81e5f814a35ed3b738ad85d4e7314b9fb24b36ecdb3828d1d4644cc8b9ce4117c862ae7baa0bd7f60362e93be2aff282a2d24c474e327d1d5948f56fff
+  data.tar.gz: 3a6235b141f139f245935b8858b175b2d54302b89dc3ac10101ffddfd10d1161760aaf0f176114a4fc6d8f0b8aa17556b8637e2f2ab815dcf875ea98c4825921

data/lib/rbbt/entity.rb

@@ -38,7 +38,7 @@ module Entity
                                                    if value.to_s == k.to_s
                                                      found = k
                                                      break
-                                                   elsif value =~ /\(#{Regexp.quote k}\)/
+                                                   elsif value.to_s =~ /\(#{Regexp.quote k}\)/
                                                      found = k
                                                      break
                                                    end

data/lib/rbbt/fix_width_table.rb

@@ -67,7 +67,7 @@ class FixWidthTable
 
   def format(pos, value)
     padding = value_size - value.length
-    if range
+    if @range
       (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
     else
       [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
@@ -105,7 +105,7 @@ class FixWidthTable
 
   def idx_value(index)
     return nil if index < 0 or index >= size
-    @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    @file.seek((@range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
     padding = @file.read(4).unpack("l").first+1
     txt = @file.read(value_size)
     str = txt.unpack(mask).first
@@ -277,7 +277,8 @@ class FixWidthTable
 
   def [](pos)
     return [] if size == 0
-    if range
+    self.read
+    if @range
       get_range(pos)
     else
       get_point(pos)
@@ -286,7 +287,7 @@ class FixWidthTable
   
   def overlaps(pos, value = false)
     return [] if size == 0
-    idxs = if range
+    idxs = if @range
       get_range(pos, true)
     else
       get_point(pos, true)

data/lib/rbbt/hpc.rb

@@ -98,6 +98,8 @@ module Marenostrum
 #SBATCH --nodes="#{nodes}"
       EOF
 
+      prep  = ""
+
       if highmem
         header +=<<-EOF
 #SBATCH --constraint=highmem 
@@ -122,7 +124,7 @@ module Marenostrum
 module load java
 
 # Calculate max available memory
-let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_ON_NODE"
+let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK"
       EOF
 
 
@@ -146,8 +148,6 @@ SINGULARITY_RUBY_INLINE="$HOME/.singularity_ruby_inline"
 mkdir -p "$SINGULARITY_RUBY_INLINE"
         EOF
 
-        prep  = ""
-
         if contain
           scratch_group_dir = File.join('/gpfs/scratch/', group)
           projects_group_dir = File.join('/gpfs/projects/', group)
@@ -172,7 +172,7 @@ cp ~/.rbbt/etc/environment $CONTAINER_DIR/.rbbt/etc/
 # Set search_paths
 echo "singularity: /singularity_opt/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" > $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "rbbt_user: /home/rbbt/.rbbt/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
-echo "home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "outside_home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "group_projects: #{projects_group_dir}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "group_scratch: #{scratch_group_dir}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "user_projects: #{projects_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
@@ -214,6 +214,8 @@ singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv tmp/ &>> #{fsync}
 EOF
+            else
+              prep = ""
             end
           end
         end

data/lib/rbbt/persist.rb

@@ -25,13 +25,18 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
-  # Return non-false if the first file is newer than the second file
-  def self.newer?(path, file)
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
+  def self.newer?(path, file, by_link = false)
     return true if not Open.exists?(file)
     path = path.find if Path === path
     file = file.find if Path === file
-    patht = Open.mtime(path)
-    filet = Open.mtime(file)
+    if by_link
+      patht = File.exists?(path) ? File.lstat(path).mtime : nil
+      filet = File.exists?(file) ? File.lstat(file).mtime : nil
+    else
+      patht = Open.mtime(path)
+      filet = Open.mtime(file)
+    end
     return true if patht.nil? || filet.nil?
     diff = patht - filet
     return diff if diff < 0

data/lib/rbbt/persist/tsv/adapter.rb

@@ -52,7 +52,6 @@ module Persist
     def lock
       return yield if @locked
       lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Log.stack caller if $LOG
       Misc.lock(lock_filename) do
         begin
           @locked = true

data/lib/rbbt/persist/tsv/fix_width_table.rb

@@ -52,7 +52,7 @@ module Persist
       if TSV::ENTRY_KEYS.include? key
         set_metadata(key, value)
       else
-        if range
+        if @range
           add_range_point key, value
         else
           add key, value
@@ -61,7 +61,7 @@ module Persist
     end
      
     def add(key, value)
-      key = pos_function.call(key) if pos_function and not (range and Array === key)
+      key = pos_function.call(key) if pos_function and not (@range and Array === key)
       super(key, value)
     end
 
@@ -85,8 +85,10 @@ module Persist
     end
 
     def each
+      read
       @size.times do |i|
-        yield i, value(i)
+        v = idx_value(i)
+        yield i, v
       end
     end
 

data/lib/rbbt/resource.rb

@@ -112,13 +112,19 @@ module Resource
               end
             when Net::HTTPRedirection, Net::HTTPFound
               location = response['location']
-              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-              FileUtils.mkdir_p final_path unless File.exist? final_path
-              TmpFile.with_file do |tmp_dir|
-                Misc.in_dir tmp_dir do
-                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+              if location.include? 'get_directory'
+                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+                FileUtils.mkdir_p final_path unless File.exist? final_path
+                TmpFile.with_file do |tmp_dir|
+                  Misc.in_dir tmp_dir do
+                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+                  end
+                  FileUtils.mv tmp_dir, final_path
+                end
+              else
+                Open.open(location, :nocache => true) do |s|
+                  Misc.sensiblewrite(final_path, s)
                 end
-                FileUtils.mv tmp_dir, final_path
               end
             when Net::HTTPInternalServerError
               @server_missing_resource_cache << url

data/lib/rbbt/resource/path.rb

@@ -297,7 +297,7 @@ module Path
 
     raise "No resource defined to produce file: #{ self }" if resource.nil?
 
-    resource.produce self, force
+    resource.produce self, force if Resource === resource
 
     self
   end

data/lib/rbbt/tsv/attach.rb

@@ -243,6 +243,7 @@ module TSV
     Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 
     if complete
+      Log.warn "Attaching through index and completing empty rows; keys with wrong format may appear (#{other.key_field} insted of #{self.key_field})" if index
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
       common_fields = (other.fields & self.fields)
@@ -255,11 +256,11 @@ module TSV
       case type
       when :single
         missing.each do |k|
-          self[k] = nil
+          self[k] = fill
         end
       when :list
         missing.each do |k|
-          values = [nil] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -267,8 +268,9 @@ module TSV
           self[k] = values
         end
       when :double
+        fill = [] if fill.nil?
         missing.each do |k|
-          values = [[]] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -276,8 +278,9 @@ module TSV
           self[k] = values
         end
       when :flat
+        fill = [] if fill.nil?
         missing.each do |k|
-          self[k] = []
+          self[k] = fill
         end
       end
     end

data/lib/rbbt/tsv/dumper.rb

@@ -32,13 +32,17 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        if fields.nil? or fields.empty?
+        if fields.nil? 
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        elsif fields.empty?
           "\n"
         else
           sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
         end
       else
-        if fields.nil? or fields.empty?
+        if fields.nil?
+          sep + values.to_s + "\n"
+        elsif fields.empty?
           "\n"
         else
           sep + values.to_s << "\n"

data/lib/rbbt/tsv/parallel.rb

@@ -2,6 +2,3 @@ require 'rbbt/util/concurrency'
 
 require 'rbbt/tsv/parallel/through'
 require 'rbbt/tsv/parallel/traverse'
-
-module TSV
-end

data/lib/rbbt/util/R.rb

@@ -41,7 +41,7 @@ source('#{UTIL}');
 
     if monitor
       #io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
-      io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true))
+      io = CMD.cmd('R --no-save --quiet', options.merge(:in => cmd, :pipe => true, :log => true, :xvfb => true))
       while line = io.gets
         case monitor
         when Proc
@@ -52,7 +52,7 @@ source('#{UTIL}');
       end
       nil
     else
-      CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd))
+      CMD.cmd('R --no-save --slave --quiet', options.merge(:in => cmd, :xvfb => true))
     end
   end
 

data/lib/rbbt/util/cmd.rb

@@ -98,7 +98,9 @@ module CMD
     post       = options.delete(:post)
     log        = options.delete(:log)
     no_fail    = options.delete(:no_fail)
+    no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
+    xvfb       = options.delete(:xvfb)
 
     dont_close_in  = options.delete(:dont_close_in)
 
@@ -116,6 +118,14 @@ module CMD
 
     end
 
+    case xvfb
+    when TrueClass
+      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
+    when String
+      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
+    when String
+    end
+
     if stderr == true
       stderr = Log::HIGH
     end

data/lib/rbbt/util/misc/bgzf.rb

@@ -7,7 +7,7 @@ module Bgzf
 
   def self.bgzip_cmd
     @@bgzip_cmd ||= begin
-                      path = `bash -c "type -p bgzips"`.strip
+                      path = `bash -c "type -p bgzip"`.strip
                       if path.empty?
                         Rbbt.claim Rbbt.software.opt.htslib, :install, Rbbt.share.install.software.HTSLIB.find(:lib)
                         Rbbt.software.opt.htslib.produce

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -61,6 +61,14 @@ module IndiferentHash
       super(key)
     end
   end
+
+  def clean_version
+    clean = {}
+    each do |k,v|
+      clean[k.to_s] = v unless clean.include? k.to_s
+    end
+    clean
+  end
 end
 
 module CaseInsensitiveHash

data/lib/rbbt/util/misc/inspect.rb

@@ -287,15 +287,19 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if obj.exists?
-              if obj.directory?
-                files = obj.glob("**/*")
-                "directory: #{Misc.fingerprint(files)}"
+            if Step === obj.resource
+              "Step file: " + obj
+            else
+              if obj.exists?
+                if obj.directory?
+                  files = obj.glob("**/*")
+                  "directory: #{Misc.fingerprint(files)}"
+                else
+                  "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
+                end
               else
-                "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
+                obj + " (file missing)"
               end
-            else
-              obj + " (file missing)"
             end
           when String
             if Misc.is_filename?(obj) and ! %w(. ..).include?(obj)

data/lib/rbbt/util/named_array.rb

@@ -117,7 +117,7 @@ module NamedArray
   #end
 
   def each(&block)
-    if defined?(Entity) and not @fields.nil? and not @fields.empty?
+    if defined?(Entity) && ! (@fields.nil? || @fields.empty?)
       i = 0
       super do |elem|
         field = @fields[i]

data/lib/rbbt/util/open.rb

@@ -16,9 +16,9 @@ module Open
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
                  ENV["GREP_CMD"]
-               elsif File.exists?('/bin/grep')
+               elsif File.exist?('/bin/grep')
                  "/bin/grep"
-               elsif File.exists?('/usr/bin/grep')
+               elsif File.exist?('/usr/bin/grep')
                  "/usr/bin/grep"
                else
                  "grep"
@@ -262,7 +262,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       remove_from_repo(*dir_sub_path)
     else
-      FileUtils.rm(file) if File.exists?(file) or Open.broken_link?(file)
+      FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
     end
   end
 
@@ -333,7 +333,7 @@ module Open
       nil
     else
       target = target.find if Path === target
-      if ! File.exists?(target)
+      if ! File.exist?(target)
         FileUtils.mkdir_p target
       end
     end
@@ -344,8 +344,8 @@ module Open
     target = target.find if Path === target
 
     target = File.join(target, File.basename(source)) if File.directory? target
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln_s source, target
   end
 
@@ -353,8 +353,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln source, target
   end
 
@@ -362,8 +362,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     begin
       CMD.cmd("ln -L '#{ source }' '#{ target }'")
     rescue ProcessFailed
@@ -475,6 +475,7 @@ module Open
       File.exist?(file) #|| File.symlink?(file)
     end
   end
+
   class << self
     alias exist? exists?
   end
@@ -747,7 +748,7 @@ module Open
     if (dir_sub_path = find_repo_dir(path))
       writable_repo?(*dir_sub_path)
     else
-      if File.exists?(path)
+      if File.exist?(path)
         File.writable?(path)
       else
         File.writable?(File.dirname(File.expand_path(path)))
@@ -776,14 +777,14 @@ module Open
       file = file.find if Path === file
       begin
         if File.symlink?(file) || File.stat(file).nlink > 1
-          if File.exists?(file + '.info') && defined?(Step)
+          if File.exist?(file + '.info') && defined?(Step)
             done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
             return done if done
           end
 
           file = Pathname.new(file).realpath.to_s 
         end
-        return nil unless File.exists?(file)
+        return nil unless File.exist?(file)
         File.mtime(file)
       rescue
         nil
@@ -793,7 +794,7 @@ module Open
 
   def self.update_mtime(path, target)
     if File.symlink?(target) || File.stat(target).nlink > 1
-      if File.exists?(target + '.info')
+      if File.exist?(target + '.info')
         target = target + '.info'
       else
         target = Pathname.new(target).realpath.to_s 
@@ -801,7 +802,7 @@ module Open
     end
 
     CMD.cmd("touch -r '#{path}' '#{target}'")
-    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exists?(path + '.info')
+    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exist?(path + '.info')
   end
 
   def self.atime(file)
@@ -823,7 +824,7 @@ module Open
   end
 
   def self.broken_link?(path)
-    File.symlink?(path) && ! File.exists?(File.readlink(path))
+    File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
   def self.download(url, path)