rbbt-util 5.21.101 → 5.21.102

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 404f1b0b70c29b749187e586ad6e213e45c672b1
-  data.tar.gz: 50248d41198d8dcd8a691a663e574a91f720a96a
+  metadata.gz: 552cc55adf8fe2641d93a6c2dd9c7c8894e53a64
+  data.tar.gz: f762f96ae17cf69f718c5eb15da13683fda1f99d
 SHA512:
-  metadata.gz: 6c91ac17825af87d3ec62e432738b494c602063c6c2a9b871d1e3a952a7b3f87b6177392899aa3f925fbb8f6687853084496bfb7c880413df0dfe68ed43eeedd
-  data.tar.gz: 3b3097806074852df41f79bbb13a7a593842be215c7f7f81373e7f78791b4512cfcf31ac5f72d3ada3f57879d7eccadc1841eedd05f8d43c62bf4948b782039d
+  metadata.gz: 53039d9cb4b69653def0d272fe3e8a6d45b058fc7123e1f79c58279d5b217e2de454c49cacee8353b8d262a603507012b7f8b4fc75e716c805df1a14155dfdf6
+  data.tar.gz: 38473fb2bd31541c0c745886c710bc484ce9b2ccd38f92af5bb1b8a85ea78a190401e046df9bf8c7f0cd6526de06aa1683dc1d2f2e9a2e82e5de4d5bd4ca76e6

data/lib/rbbt/fix_width_table.rb

@@ -84,25 +84,25 @@ class FixWidthTable
     pos(size - 1)
   end
 
-  def pos(index)
+  def idx_pos(index)
     return nil if index < 0 or index >= size
     @file.seek(5 + (record_size) * index, IO::SEEK_SET)
     @file.read(4).unpack("l").first
   end
 
-  def pos_end(index)
+  def idx_pos_end(index)
     return nil if index < 0 or index >= size
     @file.seek(9 + (record_size) * index, IO::SEEK_SET)
     @file.read(4).unpack("l").first
   end
 
-  def overlap(index)
+  def idx_overlap(index)
     return nil if index < 0 or index >= size
     @file.seek(13 + (record_size) * index, IO::SEEK_SET)
     @file.read(4).unpack("l").first
   end
 
-  def value(index)
+  def idx_value(index)
     return nil if index < 0 or index >= size
     @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
     padding = @file.read(4).unpack("l").first+1
@@ -164,7 +164,7 @@ class FixWidthTable
 
     while(upper >= lower) do
       idx = lower + (upper - lower) / 2
-      pos_idx = pos(idx)
+      pos_idx = idx_pos(idx)
 
       case pos <=> pos_idx
       when 0
@@ -183,7 +183,7 @@ class FixWidthTable
     idx.to_i
   end
 
-  def get_range(pos)
+  def get_range(pos, return_idx = false)
     case pos
     when Range
       r_start = pos.begin
@@ -197,29 +197,40 @@ class FixWidthTable
     idx = closest(r_start)
 
     return [] if idx >= size
-    return [] if idx <0 and r_start == r_end
+    return [] if idx < 0 and r_start == r_end
 
     idx = 0 if idx < 0
 
-    overlap = overlap(idx)
+    overlap = idx_overlap(idx)
 
     idx -= overlap unless overlap.nil?
 
     values = []
-    l_start = pos(idx)
-    l_end   = pos_end(idx)
-    while l_start <= r_end
-      values << value(idx) if l_end >= r_start 
-      idx += 1
-      break if idx >= size
-      l_start = pos(idx)
-      l_end   = pos_end(idx)
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    
+    if return_idx
+      while l_start <= r_end
+        values << idx if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx) if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
     end
 
     values
   end
 
-  def get_point(pos)
+  def get_point(pos, return_idx = false)
     if Range === pos
       r_start = pos.begin
       r_end   = pos.end
@@ -231,23 +242,33 @@ class FixWidthTable
     idx = closest(r_start)
 
     return [] if idx >= size
-    return [] if idx <0 and r_start == r_end
+    return [] if idx < 0 and r_start == r_end
 
     idx = 0 if idx < 0
 
-    idx += 1 unless pos(idx) >= r_start
+    idx += 1 unless idx_pos(idx) >= r_start
 
     return [] if idx >= size
 
     values = []
-    l_start = pos(idx)
-    l_end   = pos_end(idx)
-    while l_start <= r_end
-      values << value(idx)
-      idx += 1
-      break if idx >= size
-      l_start = pos(idx)
-      l_end   = pos_end(idx)
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    if return_idx 
+      while l_start <= r_end
+        values << idx
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx)
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
     end
 
     values
@@ -261,6 +282,20 @@ class FixWidthTable
       get_point(pos)
     end
   end
+  
+  def overlaps(pos, value = false)
+    return [] if size == 0
+    idxs = if range
+      get_range(pos, true)
+    else
+      get_point(pos, true)
+    end
+    if value
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx), idx_value(idx)] * ":"}
+    else
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx)] * ":"}
+    end
+  end
 
 
   def values_at(*list)

data/lib/rbbt/persist.rb

@@ -116,7 +116,7 @@ module Persist
         res
       when :array
         res = Open.read(path).split("\n", -1)
-        res.pop if res.last.empty?
+        res.pop if res.last and res.last.empty?
         res
       when :marshal
         Open.open(path) do |stream|

data/lib/rbbt/util/misc/development.rb

@@ -134,13 +134,19 @@ def self.add_libdir(dir=nil)
     $__did_once = false
   end
 
-  def self.insist(times = 3, sleep = nil, msg = nil)
+  def self.insist(times = 4, sleep = nil, msg = nil)
     if Array === times
       sleep_array = times
       times = sleep_array.length
       sleep = sleep_array.shift
     end
     try = 0
+
+    if sleep.nil?
+      sleep_array = ([0] + [0.001, 0.01, 0.1] * (times / 3)).sort[0..times-1]
+      sleep = sleep_array.shift
+    end
+
     begin
       yield
     rescue TryAgain
@@ -166,7 +172,7 @@ def self.add_libdir(dir=nil)
 
       if sleep and try > 0
         sleep sleep
-        sleep = sleep_array.shift if sleep_array
+        sleep = sleep_array.shift || sleep if sleep_array
       else
         Thread.pass
       end

data/lib/rbbt/util/misc/omics.rb

@@ -388,4 +388,48 @@ module Misc
       intersect_streams(stream1, stream2,sin, sep)
     end
   end
+
+  def self.index_BED(source, destination, sorted = false)
+
+    pos_function = Proc.new do |k|
+      k.split(":").values_at(1, 2).collect{|i| i.to_i}
+    end
+    if Open.exists? destination
+      Persist::Sharder.new destination, false, "fwt", :pos_function => pos_function  do |key|
+        key.split(":")[0]
+      end
+    else
+      io = IO === io ? io : Open.open(source) 
+
+      max_size = 0
+      nio = Misc.open_pipe do |sin|
+        while line = io.gets
+          chr, start, eend, id, *rest = line.split("\t")
+          l = id.length
+          max_size = l if max_size < l
+          chr = chr.sub('chr','')
+          sin << [chr, start, eend, id] * "\t" << "\n"
+        end
+      end
+      
+      TmpFile.with_file do |tmpfile|
+        Misc.consume_stream(nio, false, tmpfile)
+
+        value_size = max_size
+        destination = destination.find if Path === destination
+        sharder = Persist::Sharder.new destination, true, "fwt", :value_size => value_size, :range => true, :pos_function => pos_function  do |key|
+          key.split(":")[0]
+        end
+
+        TSV.traverse tmpfile, :type => :array, :bar => "Creating BED index for #{Misc.fingerprint source}" do |line|
+          chr, start, eend, id, *rest = line.split("\t")
+          key = [chr, start, eend] * ":"
+          sharder[key] = id
+        end
+        sharder.read
+
+        sharder
+      end
+    end
+  end
 end

data/lib/rbbt/util/open.rb

@@ -242,7 +242,10 @@ module Open
 
     if dir_sub_path_source.nil? and dir_sub_path_target.nil?
       FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
-      return FileUtils.mv source, target 
+      tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
+      FileUtils.mv source, tmp_target
+      FileUtils.mv tmp_target, target
+      return
     end
 
     if dir_sub_path_source.nil?

data/lib/rbbt/workflow/accessor.rb

@@ -7,6 +7,10 @@ module ComputeDependency
     dep.extend ComputeDependency
     dep.compute = value
   end
+  
+  def canfail?
+    compute == :canfail || (Array === compute && compute.include?(:canfail))
+  end
 end
 
 class Step
@@ -157,6 +161,12 @@ class Step
       i[key] = value 
       @info_cache = i
       Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
+      #Misc.insist(([0.01,0.1,1] * 3).sort) do
+      Misc.insist do
+        Open.open(info_file) do |file|
+          INFO_SERIALIAZER.load(file) 
+        end
+      end
       @info_cache_time = Time.now
       value
     end
@@ -357,9 +367,18 @@ class Step
 
   def dirty?
     status = self.status
-    return true if done? and not status == :done and not status == :noinfo
-    return true if status == :done and not done?
-    dirty_files = rec_dependencies.collect{|dependency| dependency.path unless dependency.error? and not dependency.recoverable_error? }.compact.uniq.reject{|path| ! (Path === path) || path.exists?}
+
+    if done? and not status == :done and not status == :noinfo
+      return true 
+    end
+    if status == :done and not done?
+      return true 
+    end
+
+    dirty_files = rec_dependencies.reject{|dep|
+      (dep.path && Open.exists?(dep.path)) || (dep.error? && ! dep.recoverable_error?)
+    }
+
     if dirty_files.any?
       true
     else
@@ -368,7 +387,7 @@ class Step
   end
 
   def done?
-    path and File.exist? path
+    path and Open.exists? path
   end
 
   def streaming?

data/lib/rbbt/workflow/step.rb

@@ -283,7 +283,7 @@ class Step
 
   def clean
     status = []
-    status << "dirty" if dirty?
+    status << "dirty" if done? and dirty?
     status << "not running" if not done? and not running? 
     status.unshift " " if status.any?
     Log.medium "Cleaning step: #{path}#{status * " "}"

data/lib/rbbt/workflow/step/dependencies.rb

@@ -80,13 +80,17 @@ class Step
       return if status == 'streaming' and job.running?
     end
 
-    if ((status == 'error' || job.aborted?) && job.recoverable_error?) || job.dirty? 
+    if (status == 'error' && (job.recoverable_error? || job.dirty?)) ||
+      job.aborted? ||
+      (job.done? && job.dirty?) ||
+      (status == 'waiting' && ! job.running?) 
+
       job.clean 
     end
 
     (job.init_info and job.dup_inputs) unless status == 'done' or job.started?
 
-    canfail = ComputeDependency === job and Array === job.compute and job.compute.include? :canfail
+    canfail = ComputeDependency === job && job.canfail?
     raise DependencyError, job if job.error? and not canfail
   end
 
@@ -192,13 +196,18 @@ class Step
     canfail = rest && rest.include?(:canfail)
 
     case type
+    when :canfail
+      list.each do |step|
+        step.produce
+        nil
+      end
     when :produce, :no_dup
       list.each do |step|
         Misc.insist do
           begin
             step.produce
           rescue RbbtException
-            raise $! unless canfail
+            raise $! unless canfail || step.canfail?
           rescue Exception
             step.exception $!
             if step.recoverable_error?
@@ -225,23 +234,23 @@ class Step
           begin
             dep.produce 
             Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
-          rescue Exception
-            if canfail
-              Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last}"
-            else
-              raise $!
-            end
+
           rescue Aborted
             dep.abort
             Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
             raise $!
+
           rescue Exception
-            dep.exception $!
-            dep.exception $!
-            if dep.recoverable_error?
-              raise $!
+            if canfail || dep.canfail?
+              Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last}"
             else
-              raise StopInsist.new($!)
+              Log.warn "NOT Allowing failing of #{dep.path}: #{dep.messages.last}"
+              dep.exception $!
+              if dep.recoverable_error?
+                raise $!
+              else
+                raise StopInsist.new($!)
+              end
             end
           end
         end
@@ -281,13 +290,15 @@ class Step
       end
     end
 
+    produced = []
     dependencies.each do |dep|
       next unless ComputeDependency === dep
       if dep.compute == :produce
         dep.produce 
+        produced << dep.path
       end
     end
-    
+
     self.dup_inputs
 
     required_dep_paths = []
@@ -311,6 +322,7 @@ class Step
       next unless required_dep_paths.include? step.path
       if dependencies.include?(step) and step.inputs.flatten.select{|i| Step === i}.any?
         if ComputeDependency === step
+          next if produced.include? step.path 
           compute_last_deps[step.compute] ||= []
           compute_last_deps[step.compute] << step
         else
@@ -318,6 +330,7 @@ class Step
         end
       else
         if ComputeDependency === step
+          next if produced.include? step.path 
           compute_pre_deps[step.compute] ||= []
           compute_pre_deps[step.compute] << step
         else

data/lib/rbbt/workflow/step/run.rb

@@ -352,7 +352,7 @@ class Step
       end
     end
 
-    update 
+    update if done?
 
     if dofork
       fork(true) unless started?

data/share/rbbt_commands/workflow/prov

@@ -53,9 +53,9 @@ def report_msg(status, name, path)
   workflow = Log.color(:magenta, parts.pop)
 
   if not Open.remote?(path) and (File.exists?(path) and $main_mtime and ($main_mtime - File.mtime(path)) < 0)
-    status_msg(status) << " " << [workflow, task, path] * " " << " (#{Log.color(:red, "Mtime out of sync") })\n"
+    status_msg(status.to_s) << " " << [workflow, task, path] * " " << " (#{Log.color(:red, "Mtime out of sync") })\n"
   else
-    status_msg(status) << " " << [workflow, task, path] * " " << "\n"
+    status_msg(status.to_s) << " " << [workflow, task, path] * " " << "\n"
   end
 end
 

data/test/rbbt/test_fix_width_table.rb

@@ -36,14 +36,14 @@ class TestFixWidthTable < Test::Unit::TestCase
       f.add [3,4,0], "test2"
       f.read
 
-      assert_equal 1, f.pos(0)
-      assert_equal 3, f.pos(1)
-      assert_equal 2, f.pos_end(0)
-      assert_equal 4, f.pos_end(1)
-      assert_equal 0, f.overlap(0)
-      assert_equal 0, f.overlap(1)
-      assert_equal "test1", f.value(0)
-      assert_equal "test2", f.value(1)
+      assert_equal 1, f.idx_pos(0)
+      assert_equal 3, f.idx_pos(1)
+      assert_equal 2, f.idx_pos_end(0)
+      assert_equal 4, f.idx_pos_end(1)
+      assert_equal 0, f.idx_overlap(0)
+      assert_equal 0, f.idx_overlap(1)
+      assert_equal "test1", f.idx_value(0)
+      assert_equal "test2", f.idx_value(1)
 
     end
  
@@ -104,5 +104,32 @@ g:         ____
       end
     end
   end
+
+
+  def test_range_pos
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f.overlaps(0).sort
+        assert_equal %w(1:6), f.overlaps(1).sort
+        assert_equal %w(1:6:b), f.overlaps(1, true).sort
+      end
+    end
+  end
 end
 

data/test/rbbt/util/misc/test_omics.rb

@@ -65,4 +65,32 @@ class TestMiscOmics < Test::Unit::TestCase
     assert_equal Misc.translate_prot_mutation_hgvs2rbbt("p.(A775)ins?"), nil
     assert_equal Misc.translate_prot_mutation_hgvs2rbbt("p.?del"), nil
   end
+
+  def test_index_BED
+    text= ""
+
+    %w(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y MT).each do |chr|
+      %w(1 2 3 4 5 6 7 8).each do |i|
+        start = i.to_i * 100
+        eend = start + 50
+        id = [chr, i] * ":"
+        text << [chr, start.to_s, eend.to_s, id] * "\t" + "\n"
+      end
+    end
+
+
+    io = Misc.open_pipe do |sin|
+      sin.write text
+    end
+
+    TmpFile.with_file do |dir|
+      index = Misc.index_BED(io, dir)
+      assert_equal ["1:1"], index["1:120:130"]
+      index = Misc.index_BED(io, dir)
+      assert_equal ["2:2"], index["2:220:230"]
+    end
+
+
+
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.101
+  version: 5.21.102
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-29 00:00:00.000000000 Z
+date: 2017-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake