rbbt-util 5.13.26 → 5.13.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 109a367647ca3e97bdd9cd9c7951f1dea64045d5
-  data.tar.gz: 0fc762047146427ce7dd94eddb4a8b7a05616cda
+  metadata.gz: c5b4e97dff70829b6c281c60b2ab9ef251448997
+  data.tar.gz: 7ed1a51ccbaad554e8240cdfc155eefcd0113637
 SHA512:
-  metadata.gz: 8be51c3acb036f80bba3c2cb9082244eecb1dc06892da9322ff8b1d813f99b2233bb34176590ae671a3632d031cc2c6220568f19829c4c9701907c7ec5bf0d76
-  data.tar.gz: 55a3431a1e466e2b0aeb7d742592d9a24f9a69c1c4c15697c33b675dd4cd87c3f981627c476ef38adb85100a6b811d0329bd784a31c7faa96461cd0dfd541c57
+  metadata.gz: f91cf30ebefac18ff92fd44950744d83e083a50a85e4451104f7b664bee9a1863ed8c43c95d9f0a9863e4b2ff61e92f5eb963816eff2ae25be0b6cc3f8fa7752
+  data.tar.gz: 08e361e740c52d30de3711cc142b87ebd2692e357edd3fccca917b2973b7d25dbb16a4a3d5a4db2f02180bd028b413499a775d89f27fc69069366c0a663230b0

data/lib/rbbt/monitor.rb

@@ -76,7 +76,12 @@ module Rbbt
       end
       info.values_at "pid", "ppid", "time"
     rescue Exception
-      [nil, nil, File.atime(lock)]
+      time = begin
+               File.atime(lock)
+             rescue Exception
+               Time.now
+             end
+      [nil, nil, time]
     end
   end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -4,6 +4,8 @@ module TSV
 
   def self.obj_stream(obj)
     case obj
+    when nil
+      nil
     when (defined? Step and Step)
       obj.result
     when IO, File

data/lib/rbbt/util/concurrency/processes.rb

@@ -102,7 +102,7 @@ class RbbtProcessQueue
   end
 
   def clean
-    if @process_monitor.alive? or @callback_thread.alive?
+    if (@process_monitor and @process_monitor.alive?) or (@callback_thread and @callback_thread.alive?)
       self.abort
     else
       self.join
@@ -111,8 +111,8 @@ class RbbtProcessQueue
 
   def abort
     begin
-      @process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
-      @callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
+      (@process_monitor.raise(Aborted.new); @process_monitor.join) if @process_monitor and @process_monitor.alive?
+      (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
     ensure
       join
     end

data/lib/rbbt/util/log/progress/report.rb

@@ -70,7 +70,7 @@ module Log
         eta =  (@max - @ticks) / (@ticks/used)
       end
 
-      used = [used/3600, used/60 % 60, used % 60].map{|t|  "%02i" % t }.join(':')
+      used = Misc.format_seconds(used) 
       eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
 
       indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left"

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -96,7 +96,7 @@ module ConcurrentStream
       @aborted = false if t == Thread.current
       next if t == Thread.current
       Log.medium "Aborting thread #{t.inspect}"
-      t.raise Aborted.new "From step: #{path}"
+      t.raise Aborted.new 
     end if @threads
 
     sleeped = false

data/lib/rbbt/util/misc/format.rb

@@ -18,6 +18,13 @@ module Misc
     [colors, used]
   end
 
+  def self.format_seconds(time, extended = false)
+    seconds = time.to_i
+    str = [seconds/3600, seconds/60 % 60, seconds % 60].map{|t|  "%02i" % t }.join(':')
+    str << ".%02i" % ((time - seconds) * 100) if extended
+    str
+  end
+
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/

data/lib/rbbt/util/misc/lock.rb

@@ -1,6 +1,11 @@
-Lockfile.refresh = 5 * 60 #if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
-Lockfile.max_age = 60 * 60 * 5
-Lockfile.refresh=false
+if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
+  Lockfile.refresh = false 
+  Lockfile.max_age = 60 * 60 * 5
+else
+  Lockfile.refresh = 3 
+  Lockfile.max_age = 10
+  Lockfile.suspend = 3
+end
 
 module Misc
 

data/share/rbbt_commands/system/clean

@@ -29,7 +29,7 @@ if locks.any?
     pid, ppid, time = Rbbt.load_lock(lock)
     if not Misc.pid_exists? pid
       puts "  Removing #{ lock }"
-      FileUtils.rm lock 
+      FileUtils.rm lock if File.exists? lock
     end
   end
   puts
@@ -42,8 +42,8 @@ if persists.any?
     pid, ppid, time = Rbbt.load_lock(lf)
     if not Misc.pid_exists? pid
       puts "  Removing #{ persist }"
-      FileUtils.rm persists 
-      FileUtils.rm lf 
+      FileUtils.rm persists if File.exists? persists
+      FileUtils.rm lf if File.exists? lf
     end
   end
   puts
@@ -56,8 +56,8 @@ if sensiblewrites.any?
     pid, ppid, time = Rbbt.load_lock(lf)
     if not Misc.pid_exists? pid
       puts "  Removing #{ sensiblewrite }"
-      FileUtils.rm sensiblewrite 
-      FileUtils.rm lf 
+      FileUtils.rm sensiblewrite  if File.exists? sensiblewrite
+      FileUtils.rm lf if File.exists? lf
     end
   end
   puts
@@ -69,10 +69,10 @@ puts Log.color(:magenta, "# Workflows:")
 puts
 jobs.each do |workflow, tasks|
   tasks.each do |task, jobs|
-    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": #{jobs.length}"
     done = []
     other = {}
     jobs.each do |file,h|
+      #puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": #{jobs.length}"
       status = h[:status]
       if h[:status] == :done
         done << file
@@ -85,9 +85,9 @@ jobs.each do |workflow, tasks|
       list.each do |f,p|
         if not Misc.pid_exists?(p) or status.to_s =~ /error|aborted|missing/
           puts "  Removing #{ f }"
-          FileUtils.rm_rf f
-          FileUtils.rm_rf f + '.info'
-          FileUtils.rm_rf f + '.files'
+          FileUtils.rm_rf f if File.exists? f
+          FileUtils.rm_rf f + '.info' if File.exists? f + '.info'
+          FileUtils.rm_rf f + '.files' if File.exists? f + '.files'
         end
       end
     end

data/share/rbbt_commands/system/status

@@ -13,18 +13,20 @@ Report the status of the system
 
 $ rbbt system status 
 
+-u--uncomplete Print only uncompleted or error jobs
 -h--help Print this help
 EOF
 rbbt_usage and exit 0 if options[:help]
 
+uncomplete = options.delete :uncomplete
 
 def pid_msg(pid)
-  color = if Misc.pid_exists? pid
+  color = if pid and Misc.pid_exists? pid
             :green
           else
             :red
           end
-  Log.color(color, pid)
+  Log.color(color, pid || "missing")
 end
 
 def status_msg(status)
@@ -42,7 +44,7 @@ def status_msg(status)
               nil
             end
           end
-  Log.color(color, status)
+  Log.color(color, status.to_s)
 end
 
 locks = Rbbt.locks
@@ -53,38 +55,39 @@ puts Log.color(:magenta, "# System report")
 puts
 if locks.any?
   puts Log.color(:magenta, "Locks:")
-  locks.each do |lock|
+  locks.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |lock|
     pid, ppid, time = Rbbt.load_lock(lock)
-    time ||= File.atime lock
-    puts "  " << lock + Log.color(:blue, " -- time: #{Time.now - time}; ppid: #{ppid}; pid: #{pid_msg pid}")
+    time = File.exists?(lock) ? File.ctime(lock) : Time.now
+    puts "  " << lock + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)}; ppid: #{ppid}; pid: #{pid_msg pid}")
   end
   puts
 end
 
 if persists.any?
   puts Log.color(:magenta, "Persist:")
-  persists.each do |persist|
-    puts "  " << persist
+  persists.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |persist|
+    time = File.exists?(persist) ? File.ctime(persist) : Time.now
+    puts "  " << persist + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)})")
   end
   puts
 end
 
 if sensiblewrites.any?
   puts Log.color(:magenta, "Writing:")
-  sensiblewrites.each do |sensiblewrite|
+  sensiblewrites.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |sensiblewrite|
     pid, ppid, time = Rbbt.load_lock(sensiblewrite + '.lock')
-    puts "  " << sensiblewrite + Log.color(:blue, " -- time: #{Time.now - time}; ppid: #{ppid}; pid: #{pid_msg pid}")
+    time = File.exists?(sensiblewrite) ? File.ctime(sensiblewrite) : Time.now
+    puts "  " << sensiblewrite + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)}; ppid: #{ppid}; pid: #{pid_msg pid}")
   end
   puts
 end
 
 jobs = Rbbt.jobs
 
-puts Log.color(:magenta, "# Workflows:")
+puts Log.color(:magenta, "# Workflows")
 puts
 jobs.each do |workflow, tasks|
   tasks.each do |task, jobs|
-    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": "
     done = []
     other = {}
     jobs.each do |file,h|
@@ -96,9 +99,11 @@ jobs.each do |workflow, tasks|
         other[status||"missing"] << [file, h[:pid]]
       end
     end
-    puts "  " << Log.color(:green, "done") << ": " << done.length.to_s
+    next if uncomplete and other.empty?
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": " << Log.color(:green, "done") << " " << done.length.to_s
     other.each do |status, list|
-      puts "  " << status_msg(status) << ": " << list.collect{|f,p| p.nil? ? f : f + " (#{pid_msg p})"} * ", "
+      files_txt = list.collect{|f,p| p.nil? ? f : (f + " (#{pid_msg p})") }
+      puts "  " << status_msg(status) << ": " << (files_txt * ", ")
     end
   end
 end

data/test/rbbt/util/misc/test_lock.rb

@@ -0,0 +1,31 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+
+require 'rbbt-util'
+require 'rbbt/util/misc/lock'
+
+class TestLock < Test::Unit::TestCase
+  def __test_stress
+    size = 1000000
+    num = 50
+    cpus = 200
+    TmpFile.with_file do |dir|
+      TSV.traverse (0..size).to_a, :cpus => cpus, :type => :array, :bar => true do |i|
+        begin
+          v = rand(num)
+          file = File.join(dir, "file-" << v.to_s)
+          Persist.persist("foo", :string, :file => file, :update => true) do
+            Process.pid.to_s
+          end
+
+          txt = `ls -la /proc/#{Process.pid}/fd |grep deleted`
+          Open.write(file, txt)
+          puts [Process.pid, txt.split("\n").length] * ": "
+        rescue Exception
+          Log.exception $!
+          raise $!
+        end
+      end
+    end
+  end
+end
+

data/test/rbbt/util/test_misc.rb

@@ -6,7 +6,13 @@ require 'rbbt/entity'
 
 class TestMisc < Test::Unit::TestCase
 
-  def _test_format_paragraph
+  def test_format_seconds
+    t = 61.3232
+    assert_equal "00:01:01", Misc.format_seconds(t)
+    assert_equal "00:01:01.32", Misc.format_seconds(t, true)
+  end
+
+  def test_format_paragraph
     p = <<-EOF
 Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
 incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
@@ -26,7 +32,7 @@ in culpa qui officia deserunt mollit anim id est laborum.
     assert Misc.format_paragraph(p, 70, 10, 5) =~ /\n\s*\* two/sm
   end
 
-  def _test_format_dl
+  def test_format_dl
     p1 = <<-EOF
 Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
 incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
@@ -56,13 +62,13 @@ eum fugiat quo voluptas nulla pariatur?"
     assert Misc.format_definition_list({:paragraph_first => p1, :paragraph_second => p2}) =~ /       /
   end
 
-  def _test_parse_cmd_params
+  def test_parse_cmd_params
     assert_equal ["workflow", "task", "Translation", "translate", "-f", "Associated Gene Name", "-l", "-"],
       Misc.parse_cmd_params("workflow task Translation translate -f 'Associated Gene Name' -l -")
   end
 
 
-  def _test_fixutf8
+  def test_fixutf8
     string = "abc\xffdef"
     string = string.force_encoding("UTF-8") if string.respond_to? :force_encoding
     assert(! string.valid_encoding?) if string.respond_to? :valid_encoding?
@@ -71,37 +77,37 @@ eum fugiat quo voluptas nulla pariatur?"
     assert( Misc.fixutf8(string).valid_encoding) if string.respond_to? :valid_encoding
   end
 
-  def _test_colors_for
+  def test_colors_for
     colors, used = Misc.colors_for([1,2,2,1,2,1,2,2,3,3,2,3,2])
     assert_equal Misc::COLOR_LIST[1], used[2]
   end
 
-  def _test_total_length
+  def test_total_length
     ranges = [(0..100), (50..150), (120..160)]
     ranges = [(0..100), (50..150), (120..160), (51..70)]
     assert_equal 161, Misc.total_length(ranges)
   end
 
-  def _test_id_filename?
+  def test_id_filename?
     TmpFile.with_file("") do |file|
       assert Misc.is_filename?(file)
       assert ! Misc.is_filename?("TEST STRING")
     end
   end
 
-  def _test_merge_sorted_arrays
+  def test_merge_sorted_arrays
     assert_equal [1,2,3,4], Misc.merge_sorted_arrays([1,3], [2,4])
   end
 
-  def _test_intersect_sorted_arrays
+  def test_intersect_sorted_arrays
     assert_equal [2,4], Misc.intersect_sorted_arrays([1,2,3,4], [2,4])
   end
 
-  def _test_sorted_array_matches
+  def test_sorted_array_matches
     assert_equal [1,3], Misc.sorted_array_hits(%w(a b c d e), %w(b d))
   end
 
-  def _test_binary_include?
+  def test_binary_include?
     a = %w(a b c d e).sort
     assert Misc.binary_include?(a, "a")
     assert(!Misc.binary_include?(a, "z"))
@@ -110,12 +116,12 @@ eum fugiat quo voluptas nulla pariatur?"
     assert(Misc.binary_include?(a, "d"))
   end
 
-  def _test_process_to_hash
+  def test_process_to_hash
     list = [1,2,3,4]
     assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
   end
 
-  def _test_pipe_fork
+  def test_pipe_fork
     sout, sin = Misc.pipe
     pid = Process.fork do
       Misc.purge_pipes(sin)
@@ -127,7 +133,7 @@ eum fugiat quo voluptas nulla pariatur?"
     Process.kill :INT, pid
   end
 
-  def _test_open_pipe
+  def test_open_pipe
     t = 5
     stream = Misc.open_pipe do |sin|
       t.times do |i|
@@ -148,7 +154,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }"}, lines
   end
 
-  def _test_open_pipe_fork
+  def test_open_pipe_fork
     t = 5
     stream = Misc.open_pipe(true) do |sin|
       t.times do |i|
@@ -169,7 +175,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }"}, lines
   end
 
-  def _test_open_pipe_fork_cascade
+  def test_open_pipe_fork_cascade
     t = 500
     sleep_time = 2.0 / t
     time = Time.now
@@ -205,7 +211,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal (0..t-1).to_a.collect{|i| "LINE #{ i }".reverse.downcase}, lines
   end
 
-  def _test_tee_stream
+  def test_tee_stream
     t = 500
     sleep_time = 2.0 / t
     time = Time.now
@@ -255,7 +261,7 @@ eum fugiat quo voluptas nulla pariatur?"
   end
 
 
-  def _test_string2hash
+  def test_string2hash
     assert(Misc.string2hash("--user-agent=firefox").include? "--user-agent")
     assert_equal(true, Misc.string2hash(":true")[:true])
     assert_equal(true, Misc.string2hash("true")["true"])
@@ -266,16 +272,16 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal(:j, Misc.string2hash("a=b#c=d#:h=:j")[:h])
   end
   
-  def _test_named_array
+  def test_named_array
     a = NamedArray.setup([1,2,3,4], %w(a b c d))
     assert_equal(1, a['a'])
   end
 
-  def _test_path_relative_to
+  def test_path_relative_to
     assert_equal "test/foo", Misc.path_relative_to('/test', '/test/test/foo')
   end
 
-  def _test_hash2string
+  def test_hash2string
     hash = {}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
 
@@ -293,14 +299,14 @@ eum fugiat quo voluptas nulla pariatur?"
  
   end
 
-  def _test_merge
+  def test_merge
     a = [[1],[2]]
     a = NamedArray.setup a, %w(1 2)
     a.merge [3,4]
     assert_equal [1,3], a[0]
   end
 
-  def _test_indiferent_hash
+  def test_indiferent_hash
     a = {:a => 1, "b" => 2}
     a.extend IndiferentHash
 
@@ -310,7 +316,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal 2, a[:b]
   end
 
-  def _test_lockfile
+  def test_lockfile
 
     TmpFile.with_file do |tmpfile|
       pids = []
@@ -334,7 +340,7 @@ eum fugiat quo voluptas nulla pariatur?"
     end
   end
 
-  def _test_positions2hash
+  def test_positions2hash
     inputs = Misc.positional2hash([:one, :two, :three], 1, :two => 2, :four => 4)
     assert_equal 1, inputs[:one]
     assert_equal 2, inputs[:two]
@@ -342,12 +348,12 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal nil, inputs[:four]
   end
 
-  def _test_mean
+  def test_mean
     assert_equal 2, Misc.mean([1,2,3])
     assert_equal 3, Misc.mean([1,2,3,4,5])
   end
 
-  def _test_zip_fields
+  def test_zip_fields
     current = [[:a,1], [:b,2]]
     assert_equal [[:a, :b],[1,2]], Misc.zip_fields(current)
     assert_equal current, Misc.zip_fields(Misc.zip_fields(current))
@@ -359,7 +365,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal current, Misc.zip_fields(Misc.zip_fields(current)).collect{|v| v.compact }
   end
 
-  def _test_add_zipped
+  def test_add_zipped
     current = [[:a,1], [:b,2]]
     new = %w(A B)
     Misc.append_zipped current, new
@@ -370,20 +376,20 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal Math.sqrt(2), Misc.sd([1,3])
   end
 
-  def _test_divide
+  def test_divide
     assert_equal 2, Misc.divide(%w(1 2 3 4 5 6 7 8 9),2).length
   end
 
-  def _test_ordered_divide
+  def test_ordered_divide
     assert_equal 5, Misc.ordered_divide(%w(1 2 3 4 5 6 7 8 9),2).length
   end
 
-  def _test_collapse_ranges
+  def test_collapse_ranges
     ranges = [(0..100), (50..150), (51..61),(200..250), (300..324),(320..350)]
     assert_equal [(0..150),(200..250), (300..350)], Misc.collapse_ranges(ranges)
   end
 
-  def _test_humanize
+  def test_humanize
     str1 = "test_string"
     str2 = "TEST_string"
     str3 = "test"
@@ -395,22 +401,22 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal "mutation_enrichment", Misc.snake_case("MutationEnrichment")
   end
 
-  def _test_snake_case
+  def test_snake_case
     str1 = "ACRONIMTest"
     str2 = "ACRONIM_test"
     assert_equal "ACRONIM_test", Misc.snake_case(str1)
     assert_equal "ACRONIM_test", Misc.snake_case(str2)
   end
 
-  def _test_correct_vcf_mutations
+  def test_correct_vcf_mutations
     assert_equal [737407, ["-----", "-----G", "-----GTTAAT"]], Misc.correct_vcf_mutation(737406, "GTTAAT", "G,GG,GGTTAAT")
   end
 
-  def _test_fingerprint
+  def test_fingerprint
     assert_equal '{a=>1}', Misc.fingerprint({:a => 1})
   end
 
-  def _test_tarize
+  def test_tarize
     path = File.expand_path('/home/mvazquezg/git/rbbt-util/lib')
     stream = Misc.tarize(path)
     TmpFile.with_file do |res|
@@ -420,7 +426,7 @@ eum fugiat quo voluptas nulla pariatur?"
     end
   end
 
-  def _test_camel_case
+  def test_camel_case
     assert_equal "DbSNP", Misc.camel_case("db_SNP")
     assert_equal "D3Js", Misc.camel_case("D3Js")
     assert_equal "Structure", Misc.camel_case("Structure")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.26
+  version: 5.13.27
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -313,6 +313,7 @@ files:
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -370,6 +371,7 @@ test_files:
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_R.rb