rbbt-util 5.13.1 → 5.13.2

data/lib/rbbt/tsv/util.rb

@@ -80,20 +80,6 @@ module TSV
     end
   end
 
-  def self.get_stream(file, open_options = {})
-    case
-    when Path === file
-      file.open(open_options)
-    when file.respond_to?(:gets)
-      file.rewind if file.respond_to?(:rewind) and file.eof?
-      file
-    when String === file
-      Open.open(file, open_options)
-    else
-      raise "Cannot get stream from: #{file.inspect}"
-    end
-  end
-
   def self.get_stream(file, open_options = {})
     case file
     when Path
@@ -108,8 +94,14 @@ module TSV
       raise "Could not open file given by String: #{Misc.fingerprint file}" unless Open.remote?(file) or File.exists? file
       Open.open(file, open_options)
     when (defined? Step and Step)
+      file.grace
       stream = file.get_stream
-      stream || get_stream(file.join.path)
+      if stream
+        stream
+      else
+        file.join
+        get_stream(file.path)
+      end
     when TSV::Dumper
       file.stream
     when Array

data/lib/rbbt/util/concurrency/processes.rb

@@ -29,10 +29,10 @@ class RbbtProcessQueue
             @callback.call p
           end
         rescue Aborted
-          Log.error "Callback thread aborted"
+          parent.raise $!
         rescue ClosedStream
         rescue Exception
-          Log.error "Callback thread exception"
+          Log.warn "Callback thread exception"
           parent.raise $!
         ensure
           @callback_queue.sread.close unless @callback_queue.sread.closed?
@@ -58,9 +58,9 @@ class RbbtProcessQueue
       rescue Aborted
         @processes.each{|p| p.abort }
         @processes.each{|p| p.join }
-        Log.error "Process monitor aborted"
+        Log.warn "Process monitor aborted"
       rescue Exception
-        Log.error "Process monitor exception: #{$!.message}"
+        Log.warn "Process monitor exception: #{$!.message}"
         @processes.each{|p| p.abort }
         @callback_thread.raise $! if @callback_thread
         parent.raise $!
@@ -72,7 +72,7 @@ class RbbtProcessQueue
     begin
       @callback_queue.push ClosedStream.new if @callback_thread.alive?
     rescue
-      Log.error "Error closing callback: #{$!.message}"
+      Log.warn "Error closing callback: #{$!.message}"
     end
     @callback_thread.join  if @callback_thread.alive?
   end
@@ -89,28 +89,27 @@ class RbbtProcessQueue
       Log.exception $!
       raise $!
     ensure
-      @queue.swrite.close
+      @queue.swrite.close unless @queue.swrite.closed?
     end
 
     @join.call if @join
   end
 
   def clean
-    if @process_monitor.alive?
-     @process_monitor.raise Aborted.new
-     aborted = true
-    end
-
-    if @callback_thread and @callback_thread.alive?
-     @callback_thread.raise Aborted.new
-     aborted = true
+    if @process_monitor.alive? or @callback_thread.alive?
+      self.abort
+    else
+      self.join
     end
-    raise Aborted.new if aborted
   end
 
   def abort
-    @process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
-    @callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
+    begin
+      @process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
+      @callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
+    ensure
+      join
+    end
   end
 
   def process(*e)

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -17,7 +17,9 @@ class RbbtProcessQueue
             Misc.purge_pipes
           end
 
-          Signal.trap(:INT){ raise Aborted; }
+          Signal.trap(:INT){ 
+            Kernel.exit! -1
+          }
 
           loop do
             p = @queue.pop
@@ -29,8 +31,8 @@ class RbbtProcessQueue
           end
           Kernel.exit! 0
         rescue ClosedStream
-        rescue Aborted
-          Log.error "Worker #{Process.pid} aborted"
+        rescue Aborted, Interrupt
+          Log.warn "Worker #{Process.pid} aborted"
           Kernel.exit! -1
         rescue Exception
           Log.exception $!

data/lib/rbbt/util/log.rb

@@ -110,8 +110,9 @@ module Log
 
     LOG_MUTEX.synchronize do
       STDERR.puts str
-      logfile.puts str unless logfile.nil?
       Log::LAST.replace "log"
+      logfile.puts str unless logfile.nil?
+      nil
     end
   end
 

data/lib/rbbt/util/log/progress.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/log'
 module Log
   class ProgressBar
 
-    attr_accessor :depth, :num_reports, :desc, :io, :severity, :history
+    attr_accessor :depth, :num_reports, :desc, :io, :severity, :history, :max
 
     # Creates a new instance. Max is the total number of iterations of the
     # loop. The depth represents how many other loops are above this one,
@@ -46,7 +46,6 @@ module Log
       nil
     end
 
-
     def progress
       @current.to_f/ @max
     end
@@ -104,13 +103,17 @@ module Log
     end
 
     def thr
-      @history ||= []
       @last_report = Time.now if @last_report == -1
       time = Time.now - @last_report
+      time = 0.000001 if time == 0
       thr = (@current / time).to_i
 
-      @history << thr
-      @history.shift if @history.length > 10
+      if @history.nil?
+        @history ||= []
+      else
+        @history << thr
+        @history.shift if @history.length > 10
+      end
 
       thr
     end
@@ -118,7 +121,7 @@ module Log
     def mean
       @mean_max ||= 0
       if @history.length > 3
-        mean = Misc.mean(@history[1..-1])
+        mean = Misc.mean(@history)
         @mean_max = mean if mean > @mean_max
       end
       mean
@@ -132,7 +135,7 @@ module Log
       indicator = Log.color(:magenta, @desc) 
       indicator << " #{ Log.color :blue, thr } per second"
 
-      indicator << " -- #{ Log.color :yellow, mean.to_i } avg. #{ Log.color :yellow, @mean_max.to_i} max." if mean
+      indicator << " #{ Log.color :yellow, mean.to_i } avg. #{ Log.color :yellow, @mean_max.to_i} max." if mean
 
       indicator
     end
@@ -150,7 +153,9 @@ module Log
     # original line. Everything is printed to stderr.
     def report(io = STDERR)
       if Log::LAST != "progress"
-        BARS.length.times{print(io, "\n")}
+        Log::ProgressBar.cleanup_bars
+        print(io, Log.color(:yellow, "== Progress"))
+        (BARS.length-1).times{print(io, "\n")}
       end
       print(io, up_lines(@depth) << report_msg << down_lines(@depth)) if severity >= Log.severity
       @last_report = Time.now if @last_report == -1
@@ -158,15 +163,24 @@ module Log
 
     def throughput(io = STDERR)
       if Log::LAST != "progress"
-        BARS.length.times{print(io, "\n")}
+        Log::ProgressBar.cleanup_bars
+        print(io, Log.color(:yellow, "== Progress"))
+        (BARS.length-1).times{print(io, "\n")}
       end
       print(io, up_lines(@depth) << throughput_msg << down_lines(@depth)) if severity >= Log.severity
       @last_report = Time.now
       @current = 0
     end
+
+    def done
+      print(io, up_lines(@depth) << Log.color(:magenta, @desc) << Log.color(:green, " DONE") << down_lines(@depth)) if severity >= Log.severity
+    end
+
     BAR_MUTEX = Mutex.new
     BARS = []
+    REMOVE = []
     def self.new_bar(max, options = {})
+      cleanup_bars
       BAR_MUTEX.synchronize do
         options = Misc.add_defaults options, :depth => BARS.length
         BARS << (bar = ProgressBar.new(max, options))
@@ -174,18 +188,27 @@ module Log
       end
     end
 
-    def self.remove_bar(bar)
+    def self.cleanup_bars
       BAR_MUTEX.synchronize do
-        index = BARS.index bar
-        if index
-          (index+1..BARS.length-1).each do |pos|
-            bar = BARS[pos]
-            bar.depth = pos - 1
-            BARS[pos-1] = bar
-          end 
-          BARS.pop
+        REMOVE.each do |bar|
+          index = BARS.index bar
+          if index
+            (index..BARS.length-1).each do |pos|
+              bar = BARS[pos]
+              bar.depth = pos - 1
+              BARS[pos-1] = bar
+            end 
+            BARS.pop
+          end
         end
-        Log::LAST.replace "removeprogress"
+        REMOVE.clear
+      end
+    end
+
+    def self.remove_bar(bar)
+      BAR_MUTEX.synchronize do
+        bar.done unless bar.max
+        REMOVE << bar
       end
     end
 
@@ -193,8 +216,11 @@ module Log
       bar = new_bar(max, options)
       begin
         yield bar
+        keep = false
+      rescue KeepBar
+        keep = true
       ensure
-        remove_bar(bar)
+        remove_bar(bar) unless bar
       end
     end
   end

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,6 +1,10 @@
 class ParameterException < Exception; end
 class FieldNotFoundError < Exception;end
-class Aborted < Exception; end
+class Aborted < Exception
+  def initialize(*args)
+    super(*args)
+  end
+end
 class TryAgain < Exception; end
 class ClosedStream < Exception; end
 class ProcessFailed < Exception; end
@@ -10,3 +14,9 @@ class KeepLocked < Exception
     @payload = payload
   end
 end
+class KeepBar < Exception
+  attr_accessor :payload
+  def initialize(payload)
+    @payload = payload
+  end
+end

data/lib/rbbt/util/misc/inspect.rb

@@ -42,7 +42,7 @@ module Misc
     when (defined? TSV and TSV::Parser)
       "<TSVStream:" + (obj.filename || "NOFILENAME") + "--" << Misc.fingerprint(obj.options) << ">"
     when IO
-      "<IO:" + (obj.respond_to?(:filename) ? obj.filename || obj.inspect : obj.inspect) + ">"
+      (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect) + ">" : obj.inspect
     when File
       "<File:" + obj.path + ">"
     when Array

data/lib/rbbt/util/misc/lock.rb

@@ -22,7 +22,7 @@ module Misc
               raise "No info" unless info
 
               if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
-                Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+                Log.high("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
                 FileUtils.rm lock_path
               end
             end

data/lib/rbbt/util/misc/pipes.rb

@@ -124,22 +124,22 @@ module Misc
           begin 
             stream_in1.write block; 
           rescue IOError
-            Log.error("Tee stream 1 #{stream} IOError: #{$!.message}");
+            Log.warn("Tee stream 1 #{Misc.fingerprint stream} IOError: #{$!.message}");
             skip1 = true
           end unless skip1 
 
           begin 
             stream_in2.write block
           rescue IOError
-            Log.error("Tee stream 2 #{stream} IOError: #{$!.message}");
+            Log.warn("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
             skip2 = true
           end unless skip2 
         end
         stream_in1.close unless stream_in1.closed?
         stream_in2.close unless stream_in2.closed?
         stream.join if stream.respond_to? :join
-      rescue Aborted
-        Log.error("Tee stream #{stream} Aborted");
+      rescue Aborted, Interrupt
+        Log.warn("Tee stream #{Misc.fingerprint stream} Aborted");
         stream.abort if stream.respond_to? :abort
       rescue Exception
         Log.exception $!
@@ -177,6 +177,8 @@ module Misc
       io.join if io.respond_to? :join
       return
     end
+
+    Log.medium "Consuming stream #{Misc.fingerprint io}"
     if in_thread
       Thread.new do
         consume_stream(io, false)
@@ -188,9 +190,9 @@ module Misc
           Thread.pass 
         end
         io.join if io.respond_to? :join
+      rescue IOError
       rescue
-        Log.error "Exception consuming stream: #{io.inspect}"
-        Log.exception $!
+        Log.warn "Exception consuming stream: #{io.inspect}"
         io.abort if io.respond_to? :abort
       end
     end
@@ -240,9 +242,13 @@ module Misc
 
           Open.mv tmp_path, path
         rescue Aborted
-          Log.error "Aborted sensiblewrite -- #{ Log.color :blue, path }"
+          Log.warn "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
+          content.abort if content.respond_to? :abort
+          content.join if content.respond_to? :join
         rescue Exception
-          Log.error "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
+          Log.warn "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
+          content.abort if content.respond_to? :abort
+          content.join if content.respond_to? :join
           Open.rm_f path if File.exists? path
           raise $!
         ensure
@@ -264,29 +270,40 @@ module Misc
 
   def self.sort_stream(stream, header_hash = "#")
     Misc.open_pipe do |sin|
-      line = stream.gets
-      while line =~ /^#{header_hash}/ do
-        sin.puts line
+      begin
+        if defined? Step and Step === stream
+          step = stream
+          stream = stream.get_stream || stream.path.open
+        end
+
         line = stream.gets
-      end
+        while line =~ /^#{header_hash}/ do
+          sin.puts line
+          line = stream.gets
+        end
 
-      line_stream = Misc.open_pipe do |line_stream_in|
-        begin
-          while line
-            line_stream_in.puts line
-            line = stream.gets
+        line_stream = Misc.open_pipe do |line_stream_in|
+          begin
+            while line
+              line_stream_in.puts line
+              line = stream.gets
+            end
+            stream.join if stream.respond_to? :join
+          rescue
+            stream.abort if stream.respond_to? :abort
+            raise $!
           end
-          stream.join if stream.respond_to? :join
-        rescue
-          stream.abort if stream.respond_to? :abort
-          raise $!
         end
-      end
 
-      sorted = CMD.cmd("sort", :in => line_stream, :pipe => true)
+        sorted = CMD.cmd("sort", :in => line_stream, :pipe => true)
 
-      while block = sorted.read(2048)
-        sin.write block
+        while block = sorted.read(2048)
+          sin.write block
+        end
+      rescue
+        if defined? step and step
+          step.abort
+        end
       end
     end
   end
@@ -330,6 +347,14 @@ module Misc
     num_streams = streams.length
     Misc.open_pipe do |sin|
       sin.puts header if header
+      streams = streams.collect do |stream|
+        if defined? Step and Step === stream
+          stream.get_stream || stream.join.path.open
+        else
+          stream
+        end
+      end
+
       begin
         done_streams = []
         lines ||= streams.collect{|s| s.gets }
@@ -369,6 +394,7 @@ module Misc
           stream.join if stream.respond_to? :join
         end
       rescue 
+        Log.exception $!
         streams.each do |stream|
           stream.abort if stream.respond_to? :abort
         end
@@ -388,4 +414,19 @@ module Misc
     tee2
   end
 
+  def self.save_stream(file, stream)
+    out, save = Misc.tee_stream stream
+
+    Thread.new(Thread.current) do |parent|
+      begin
+        Misc.sensiblewrite(file, save)
+      rescue
+        save.abort if save.respond_to? :abort
+        parent.raise $!
+      end
+    end
+
+    out
+  end
+
 end