rbbt-util 5.12.3 → 5.13.0

data/lib/rbbt/util/simpleopt/get.rb

@@ -26,7 +26,7 @@ module SOPT
         values[input] = value
       else
         if value.nil? and %w(F false FALSE no).include?(args[i])
-          Log.warn "Boolean values must are best specified as #{current}[=true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value" 
+          Log.warn "Boolean values must are best specified as #{current}=[true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value" 
           value = args.delete_at(i)
         end
         values[input] = %w(F false FALSE no).include?(value)? false : true

data/lib/rbbt/workflow/accessor.rb

@@ -104,9 +104,41 @@ class Step
     set_info(:messages, (messages || []) << message)
   end
 
-  def self.log(status, message, path, &block)
-    if block_given?
-      start = Time.now
+  def self.log_block(staus, message, path, &block)
+    start = Time.now
+    status = status.to_s
+    status_color = case status
+                   when "starting"
+                     :yellow
+                   when "error"
+                     :red
+                   when "done"
+                     :green
+                   else
+                     :cyan
+                   end
+    Log.info do 
+      now = Time.now
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str
+    end
+    res = yield
+    eend = Time.now
+    Log.info do 
+      now = Time.now
+      str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.1g" % (eend - start)}"
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str
+    end
+    res
+  end
+
+  def self.log_string(status, message, path)
+    Log.info do 
+
       status = status.to_s
       status_color = case status
                      when "starting"
@@ -118,43 +150,32 @@ class Step
                      else
                        :cyan
                      end
-      Log.info do 
-        now = Time.now
-        str = Log.color :reset
-        str << "#{ Log.color status_color, status}"
-        str << ": #{ message }" if message
-        str << " -- #{Log.color :blue, path.to_s}" if path
-        str
-      end
-      res = yield
-      eend = Time.now
-      Log.info do 
-        now = Time.now
-        str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.1g" % (eend - start)}"
-        str << " -- #{Log.color :blue, path.to_s}" if path
-        str
+
+      str = Log.color :reset
+      str << "#{ Log.color status_color, status}"
+      str << ": #{ message }" if message
+      str << " -- #{Log.color :blue, path.to_s}" if path
+      str
+    end
+  end
+
+  def self.log_progress(status, options, path, &block)
+    options = Misc.add_defaults options, :severity => Log::INFO
+    max = Misc.process_options options, :max
+    Log::ProgressBar.with_bar(max, options) do |bar|
+      yield bar
+    end
+  end
+
+  def self.log(status, message, path, &block)
+    if block_given?
+      if Hash === message
+        log_progress(status, message, path, &block)
+      else
+        log_block(status, message, path, &block)
       end
-      res
     else
-      status = status.to_s
-      status_color = case status
-                     when "starting"
-                       :yellow
-                     when "error"
-                       :red
-                     when "done"
-                       :green
-                     else
-                       :cyan
-                     end
-      Log.info do 
-        now = Time.now
-        str = Log.color :reset
-        str << "#{ Log.color status_color, status}"
-        str << ": #{ message }" if message
-        str << " -- #{Log.color :blue, path.to_s}" if path
-        str
-      end
+      log_string(status, message, path)
     end
   end
 

data/lib/rbbt/workflow/step/run.rb

@@ -61,7 +61,6 @@ class Step
 
   def _exec
     @exec = true if @exec.nil?
-    #@task.exec_in((bindings ? bindings : self), *@inputs)
     @task.exec_in((bindings ? bindings : self), *dup_inputs)
   end
 
@@ -242,7 +241,7 @@ class Step
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
         begin
-          res = run
+          res = run true
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Aborted"
@@ -251,7 +250,7 @@ class Step
           Log.debug("Exception '#{$!.message}' caught on forked process: #{path}")
           raise $!
         ensure
-          join
+          join_stream
         end
 
         begin

data/share/rbbt_commands/workflow/task

@@ -337,8 +337,12 @@ begin
   end
 
   if options.delete(:printpath)
-    job.join.load
-    puts job.path
+    job.join
+    if Open.remote? job.path
+      puts job.path + Log.color(:blue, "?_format=raw")
+    else
+      puts job.path
+    end
     exit 0
   end
 
@@ -378,7 +382,13 @@ begin
     space.times do
       Log.clear_line 
     end
-    out.puts job.path
+
+    if Open.remote? job.path
+      out.puts job.path + Log.color(:blue, "?_format=raw")
+    else
+      out.puts job.path
+    end
+
     exit 0
   end
 rescue ParameterException
@@ -412,7 +422,7 @@ when Step
       Thread.pass while IO.select([io]).nil?
       while block = io.read(2048) do
         out.write block
-      end unless io.closed? 
+      end #unless io.closed? 
       io.join if io.respond_to? :join
     rescue Exception
       Log.exception $!

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -283,7 +283,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     require 'rbbt/sources/organism'
 
     head = 10_000
-    cpus = nil
+    cpus = 4
 
     stream = Organism.identifiers("Hsa").open 
     dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
@@ -337,4 +337,38 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
     assert_equal size, stream.read.split("\n").length
   end
+
+  def test_traverse_progress
+    size = 1000
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
+      sleep 0.001
+      e
+    end
+    assert_equal size, stream.read.split("\n").length
+
+    size = 1000
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
+      sleep 0.001
+      e
+    end
+    assert_equal size, stream.read.split("\n").length
+
+    size = 1000
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
+      sleep 0.001
+      e
+    end
+    assert_equal size, stream.read.split("\n").length
+
+    size = 1000
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
+      sleep 0.01
+      e
+    end
+    assert_equal size, stream.read.split("\n").length
+  end
 end

data/test/rbbt/util/log/test_progress.rb

@@ -0,0 +1,49 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/log/progress'
+
+class TestProgress < Test::Unit::TestCase
+  def _test_bar
+    t1 = Thread.new do
+      Log::ProgressBar.with_bar(20) do |bar|
+        20.times do
+          bar.tick
+          sleep 0.3
+        end
+      end
+    end
+
+    t2 = Thread.new do
+      Log::ProgressBar.with_bar(20) do |bar|
+        20.times do
+          bar.tick
+          sleep 0.2
+        end
+      end
+    end
+    t1.join
+    t2.join
+  end
+
+  def test_bar_no_size
+    t1 = Thread.new do
+      Log::ProgressBar.with_bar(nil) do |bar|
+        20.times do
+          bar.tick
+          sleep 0.3
+        end
+      end
+    end
+
+    t2 = Thread.new do
+      Log::ProgressBar.with_bar(nil) do |bar|
+        20.times do
+          bar.tick
+          sleep 0.2
+        end
+      end
+    end
+    t1.join
+    t2.join
+  end
+end
+

data/test/rbbt/util/misc/test_pipes.rb

@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
 
 class TestMiscPipes < Test::Unit::TestCase
 
-  def _test_collapse_stream
+  def test_collapse_stream
     text=<<-EOF
 row1 A B C
 row1 a b c
@@ -19,7 +19,7 @@ row2 aa bb cc
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end
 
-  def _test_paste_stream
+  def test_paste_stream
     text1=<<-EOF
 row1 A B C
 row2 AA BB CC
@@ -46,7 +46,7 @@ row3 ccc
     assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
   end
 
-  def _test_sort_stream
+  def test_sort_stream
     text =<<-EOF
 #: :sep=" "
 #Row LabelA LabelB LabelC
@@ -60,7 +60,7 @@ row1 A B C
     assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
-  def _test_dup_stream
+  def test_dup_stream
     text =<<-EOF
 #: :sep=" "
 #Row LabelA LabelB LabelC

data/test/rbbt/util/test_open.rb

@@ -122,7 +122,6 @@ class TestOpen < Test::Unit::TestCase
         end
       end
 
-
       Misc.benchmark(100) do
         filename = "file" << (rand * 100).to_i.to_s
         Open.write(File.join(repo, filename), file2)
@@ -130,8 +129,6 @@ class TestOpen < Test::Unit::TestCase
           Open.read(File.join(repo, filename))
         end
       end
-
-
     end
   end
   

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.12.3
+  version: 5.13.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-24 00:00:00.000000000 Z
+date: 2014-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -174,6 +174,7 @@ files:
 - lib/rbbt/util/excel2tsv.rb
 - lib/rbbt/util/filecache.rb
 - lib/rbbt/util/log.rb
+- lib/rbbt/util/log/progress.rb
 - lib/rbbt/util/misc.rb
 - lib/rbbt/util/misc/concurrent_stream.rb
 - lib/rbbt/util/misc/development.rb
@@ -188,6 +189,7 @@ files:
 - lib/rbbt/util/misc/omics.rb
 - lib/rbbt/util/misc/options.rb
 - lib/rbbt/util/misc/pipes.rb
+- lib/rbbt/util/misc/progress.rb
 - lib/rbbt/util/misc/system.rb
 - lib/rbbt/util/named_array.rb
 - lib/rbbt/util/open.rb
@@ -297,6 +299,7 @@ files:
 - test/rbbt/util/concurrency/processes/test_socket.rb
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -357,6 +360,7 @@ test_files:
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_R.rb
+- test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_excel2tsv.rb