rbbt-util 5.13.19 → 5.13.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b571306a9f5980baaa8c5d13939442178440a9e
-  data.tar.gz: de34e0b79dc8e1736732e64d42af40d17f2ad53f
+  metadata.gz: 66e0264c3104ab53b278845aaab51e2ae75b5d40
+  data.tar.gz: 7abc522ee044b2527b0a6a40acff4a8fde3c7c90
 SHA512:
-  metadata.gz: 6e13708123ad0a40e9bc2dfc99cbdc6c67959f2a39aa5f8c5ac512cd933775374f1195f3d5b7a127787c4eafbf99bd18bd248b296957ae28974d51041563f5a5
-  data.tar.gz: 034c6c8e537be9f35f8c5630f2dd40810747bdda97126303f7737adc2d7cec2d7e2e19c31568f64caf0bddffaabc6762c32fdbdfaca8bb2eebb05210bf31e891
+  metadata.gz: 925b812e86aed64d95cb08b6f56491518640953f8284168bbefd21e5500989b9a6a8755df667d552115f908a5e4913b4d9c5fac3b9fba2214c66e0162c492c1e
+  data.tar.gz: 0af9ab272db85087c7c2244d894a0af2c68867c93117d6d1e594bb676b59978bdb1fa5790e915204be8a2423187713b344b02d56af19abcd9ac19d146bc61b6c

data/lib/rbbt/tsv/stream.rb

@@ -74,6 +74,7 @@ module TSV
 
       lines = []
       fields = []
+      sizes = []
       key_fields = []
       input_options = []
       empty = []
@@ -84,6 +85,7 @@ module TSV
         empty << stream if parser.first_line.nil?
         key_fields << parser.key_field
         fields << parser.fields
+        sizes << parser.fields.length
         input_options << parser.options
 
         parser.stream
@@ -113,7 +115,6 @@ module TSV
           keys[i] = key
           parts[i] = p
         end
-        sizes = parts.collect{|p| p.length }
         last_min = nil
         while lines.compact.any?
           min = keys.compact.sort.first

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -12,7 +12,7 @@ class RbbtProcessQueue
       key = "/" << rand(100000000).to_s;
       @write_sem = key + '.in'
       @read_sem = key + '.out'
-      Log.warn "Creating socket semaphores: #{key}"
+      Log.low "Creating socket semaphores: #{key}"
       RbbtSemaphore.create_semaphore(@write_sem,1)
       RbbtSemaphore.create_semaphore(@read_sem,1)
     end
@@ -20,7 +20,7 @@ class RbbtProcessQueue
     def clean
       @sread.close unless @sread.closed?
       @swrite.close unless @swrite.closed?
-      Log.warn "Destroying socket semaphores"
+      Log.low "Destroying socket semaphores"
       RbbtSemaphore.delete_semaphore(@write_sem)
       RbbtSemaphore.delete_semaphore(@read_sem)
     end

data/lib/rbbt/util/log.rb

@@ -6,6 +6,8 @@ module Log
   extend Term::ANSIColor
 
 
+  LOG_MUTEX = Mutex.new
+
   SEVERITY_NAMES ||= begin
                      names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR ) 
                      names.each_with_index do |name,i|
@@ -14,6 +16,20 @@ module Log
                      names
                    end
 
+  def self.ignore_stderr
+    backup_stderr = STDERR.dup
+    LOG_MUTEX.synchronize do
+      begin
+        File.open('/dev/null', 'w') do |f|
+          STDERR.reopen(f)
+          yield
+        end
+      ensure
+        STDERR.reopen backup_stderr
+      end
+    end
+  end
+
   def self.get_level(level)
     case level
     when Fixnum
@@ -35,8 +51,11 @@ module Log
 
   self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
 
-  require "highline/system_extensions.rb"
-  self.tty_size = HighLine::SystemExtensions.terminal_size.first 
+  self.ignore_stderr do
+    require 'nokogiri'
+    require "highline/system_extensions.rb"
+    self.tty_size = HighLine::SystemExtensions.terminal_size.first 
+  end
 
   def self.with_severity(level)
     orig = Log.severity
@@ -95,7 +114,6 @@ module Log
     end
   end
 
-  LOG_MUTEX = Mutex.new
   LAST = "log"
   def self.log(message = nil, severity = MEDIUM, &block)
     return if severity < self.severity 

data/lib/rbbt/workflow/step/run.rb

@@ -21,11 +21,6 @@ class Step
           else
             Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
             Misc.dup_stream(current)
-            #STREAM_CACHE[stream] = new
-            #new
-            #new = current.dup
-            #new.rewind
-            #new
           end
 
         else
@@ -145,7 +140,7 @@ class Step
           next if dependency.running?
           dependency.clean 
         else
-          dependency.clean if (dependency.error? or dependency.aborted? or not dependency.done?)
+          dependency.clean if (dependency.error? or dependency.aborted? or dependency.status.nil?)# or not dependency.done?)
         end
 
         unless dependency.result or dependency.done?
@@ -178,7 +173,7 @@ class Step
 
           Open.rm info_file if Open.exists? info_file
 
-          log :setup, "#{Log.color :magenta, "Task"} #{Log.color :yellow, task.name.to_s || ""}"
+          log :setup, "#{Log.color :cyan, "Setup"} #{Log.color :yellow, task.name.to_s || ""}"
 
           merge_info({
             :pid => Process.pid,
@@ -199,13 +194,13 @@ class Step
           set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
           set_info :started, (start_time = Time.now)
-          log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""}"
+          log :started, "#{Log.color :magenta, "Starting"} #{Log.color :yellow, task.name.to_s || ""}"
 
           begin
             result = _exec
           rescue Aborted
             stop_dependencies
-            log(:error, "Aborted")
+            log(:aborted, "Aborted")
             raise $!
           rescue Exception
             backtrace = $!.backtrace
@@ -224,16 +219,21 @@ class Step
             result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
           end
 
-          case result
-          when IO
+          stream = case result
+                   when IO
+                     result
+                   when TSV::Dumper
+                     result.stream
+                   end
 
-            log :streaming, "#{Log.color :magenta, "Streaming IO"} #{Log.color :yellow, task.name.to_s || ""}"
-            ConcurrentStream.setup result do
+          if stream
+            log :streaming, "#{Log.color :magenta, "Streaming"} #{Log.color :yellow, task.name.to_s || ""}"
+            ConcurrentStream.setup stream do
               begin
                 if status != :done
                   set_info :done, (done_time = Time.now)
                   set_info :time_elapsed, (time_elapsed = done_time - start_time)
-                  log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+                  log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
                 end
               rescue
                 Log.exception $!
@@ -241,7 +241,7 @@ class Step
                 join
               end
             end
-            result.abort_callback = Proc.new do
+            stream.abort_callback = Proc.new do
               begin
                 log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
               rescue
@@ -250,32 +250,10 @@ class Step
                 join
               end
             end
-          when TSV::Dumper
-            log :streaming, "#{Log.color :magenta, "Streaming TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""}"
-            ConcurrentStream.setup result.stream do
-              begin
-                if status != :done
-                  set_info :done, (done_time = Time.now)
-                  set_info :time_elapsed, (time_elapsed = done_time - start_time)
-                  log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
-                end
-              rescue
-                Log.exception $!
-              ensure
-                join
-              end
-            end
-            result.stream.abort_callback = Proc.new do
-              begin
-                log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}"  if status == :streaming
-              rescue Exception
-                Log.exception $!
-              end
-            end
           else
             set_info :done, (done_time = Time.now)
             set_info :time_elapsed, (time_elapsed = done_time - start_time)
-            log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+            log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
           end
 
           result
@@ -470,6 +448,7 @@ class Step
         dependencies.each{|dep| dep.join }
       end
       self
+      sleep 1 until path.exists?
     ensure
       set_info :joined, true
     end

data/share/rbbt_commands/benchmark/throughput

@@ -23,7 +23,7 @@ max = 0
 avg = 0
 all = []
 scale = 5
-bar = Log::ProgressBar.new_bar nil
+bar = Log::ProgressBar.new_bar nil, :desc => "Throughput"
 
 while line = STDIN.gets
   bar.tick

data/share/rbbt_commands/workflow/cmd

@@ -6,8 +6,9 @@ require 'rbbt/workflow'
 
 workflow = ARGV.shift
 
-Workflow.require_workflow workflow
-dir = $command_dir = Workflow.workflows.last.libdir.share.rbbt_commands
+wf = Workflow.require_workflow workflow
+dir = $command_dir = wf.libdir.share.rbbt_commands
+
 def commands(prev)
     rbbt_command_dir = $command_dir
 

data/test/rbbt/tsv/test_stream.rb

@@ -89,4 +89,37 @@ row2 cc
     assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
     assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
   end
+
+  def test_paste_stream_missing
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row1 a b
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "a", "b", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
 end

data/test/rbbt/util/test_log.rb

@@ -14,8 +14,10 @@ class TestLog < Test::Unit::TestCase
     assert Log.color(:green, "green")
   end
 
-  def test_return_line
-    puts Log.return_line  + "ccc"*3
+  def test_no_stderr
+    Log.ignore_stderr do
+      STDERR.puts "NOPRINT"
+    end
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.19
+  version: 5.13.20
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake