rbbt-util 5.12.3 → 5.13.0

data/lib/rbbt/util/concurrency/processes.rb

@@ -4,11 +4,12 @@ require 'rbbt/util/concurrency/processes/socket'
 class RbbtProcessQueue
   #{{{ RbbtProcessQueue
 
-  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup
-  def initialize(num_processes, cleanup = nil)
+  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join
+  def initialize(num_processes, cleanup = nil, join = nil)
     @num_processes = num_processes
     @processes = []
     @cleanup = cleanup
+    @join = join
     @queue = RbbtProcessSocket.new
   end
 
@@ -90,6 +91,8 @@ class RbbtProcessQueue
     ensure
       @queue.swrite.close
     end
+
+    @join.call if @join
   end
 
   def clean

data/lib/rbbt/util/concurrency/threads.rb

@@ -4,26 +4,28 @@ class RbbtThreadQueue
   class RbbtThreadQueueWorker < Thread
     def initialize(queue, mutex = nil, &block)
       if mutex.nil?
-        super(Thread.current) do |current|
+        super(Thread.current) do |parent|
           begin
             loop do
               p = queue.pop
               block.call *p
             end
+          rescue Aborted
           rescue Exception
-            current.raise $! unless Aborted === $!
+            parent.raise $! 
           end
         end
       else
-        super(Thread.current) do |current|
+        super(Thread.current) do |parent|
           begin
             loop do
               p = queue.pop
               p = Array === p ? p << mutex : [p,mutex]
               block.call *p
             end
+          rescue Aborted
           rescue Exception
-            current.raise $! unless Aborted === $!
+            parent.raise $! 
           end
         end
       end

data/lib/rbbt/util/log.rb

@@ -1,5 +1,6 @@
 require 'term/ansicolor'
 require 'rbbt/util/color'
+require 'rbbt/util/log/progress'
 
 module Log
   extend Term::ANSIColor

data/lib/rbbt/util/log/progress.rb

@@ -0,0 +1,163 @@
+require 'rbbt/util/log'
+module Log
+  class ProgressBar
+
+    attr_accessor :depth, :num_reports, :desc, :io, :severity
+
+    # Creates a new instance. Max is the total number of iterations of the
+    # loop. The depth represents how many other loops are above this one,
+    # this information is used to find the place to print the progress
+    # report.
+    def initialize(max, options = {})
+      options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity
+      depth, num_reports, desc, io, severity = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity
+
+      @max = max
+      @max = 1 if @max and @max < 1
+      @current = 0
+      @time = Time.now
+      @last_report = -1
+      @num_reports = num_reports
+      @severity = severity
+      @depth = depth
+      @desc = desc
+    end
+
+    # Used to register a new completed loop iteration.
+    def tick(step = nil)
+
+      if step.nil?
+        @current += 1
+      else
+        @current = step
+      end
+
+      if @max
+        if percent - @last_report > 1.to_f/@num_reports.to_f 
+          report
+          @last_report=percent
+        end
+      else
+        @last_report = Time.now if @last_report == -1
+        if Time.now - @last_report >= 1.0 
+          throughput
+        end
+      end
+
+      nil
+    end
+
+
+    def progress
+      @current.to_f/ @max
+    end
+
+    def percent
+      (self.progress * 100).to_i
+    end
+
+    def eta
+      (Time.now - @time)/progress * (1-progress)
+    end
+
+    def used
+      (Time.now - @time).to_i
+    end
+
+
+    def up_lines(depth)
+      "\033[#{depth + 2}F\033[2K"
+    end
+
+    def down_lines(depth)
+      "\n\033[#{depth + 3}E"
+    end
+
+    def report_msg
+      progress = self.progress
+      percent = self.percent
+
+      indicator = Log.color(:magenta, @desc) << " "
+      10.times{|i|
+        if i < progress * 10 then
+          indicator << Log.color(:yellow, ".")
+        else
+          indicator << " "
+        end
+      }
+      done = progress == 1
+
+      used = self.used
+      used = [used/3600, used/60 % 60, used % 60].map{|t|  "%02i" % t }.join(':')
+
+      if progress == 1
+        indicator << Log.color(:green, " done ")
+
+
+        indicator << Log.color(:blue, " #{used}")
+      else
+        indicator << " done #{Log.color(:blue, percent.to_s << "%")}"
+
+        eta = self.eta
+        eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
+
+        indicator << " (Time left #{eta} seconds) (Started #{used} seconds ago)"
+      end
+
+    end
+
+    def throughput_msg
+      indicator = Log.color(:magenta, @desc) 
+      time = Time.now - @last_report
+      thr = (@current / time).to_i
+      indicator << " #{ Log.color :blue, thr } per second"
+      indicator
+    end
+
+    # Prints de progress report. It backs up as many lines as the meters
+    # depth. Prints the progress as a line of dots, a percentage, time
+    # spent, and time left. And then goes moves the cursor back to its
+    # original line. Everything is printed to stderr.
+    def report(io = STDERR)
+      io.print(up_lines(@depth) << report_msg << down_lines(@depth)) if severity >= Log.severity
+    end
+
+    def throughput(io = STDERR)
+      io.print(up_lines(@depth) << throughput_msg << down_lines(@depth)) if severity >= Log.severity
+      @last_report = Time.now
+      @current = 0
+    end
+    BAR_MUTEX = Mutex.new
+    BARS = []
+    def self.new_bar(max, options = {})
+      options = Misc.add_defaults options, :depth => BARS.length
+      BAR_MUTEX.synchronize do
+        BARS << (bar = ProgressBar.new(max, options))
+        bar
+      end
+    end
+
+    def self.remove_bar(bar)
+      BAR_MUTEX.synchronize do
+        index = BARS.index bar
+        if index
+          (index+1..BARS.length-1).each do |pos|
+            bar = BARS[pos]
+            bar.depth = pos - 1
+            BARS[pos-1] = bar
+          end 
+          BARS.pop
+        end
+      end
+    end
+
+    def self.with_bar(max, options = {})
+      bar = new_bar(max, options)
+      begin
+        yield bar
+      ensure
+        remove_bar(bar)
+      end
+    end
+  end
+end

data/lib/rbbt/util/misc/options.rb

@@ -110,6 +110,7 @@ module Misc
   end
 
   def self.add_defaults(options, defaults = {})
+    options ||= {}
     case
     when Hash === options
       new_options = options.dup

data/lib/rbbt/util/misc/pipes.rb

@@ -73,65 +73,74 @@ module Misc
     sout
   end
 
-  def self.tee_stream_fork(stream)
-    stream_out1, stream_in1 = Misc.pipe
-    stream_out2, stream_in2 = Misc.pipe
-
-    splitter_pid = Process.fork do
-      Misc.purge_pipes(stream_in1, stream_in2)
-      stream_out1.close
-      stream_out2.close
-      begin
-        filename = stream.respond_to?(:filename)? stream.filename : nil
-        skip1 = skip2 = false
-        while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
-          begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
-        end
-        raise "Error writing in stream_in1" if skip1
-        raise "Error writing in stream_in2" if skip2
-        stream.join if stream.respond_to? :join
-        stream_in1.close 
-        stream_in2.close 
-      rescue Aborted
-        stream.abort if stream.respond_to? :abort
-        raise $!
-      rescue IOError
-        Log.exception $!
-      rescue Exception
-        Log.exception $!
-      end
-    end
-    stream.close
-    stream_in1.close
-    stream_in2.close
-
-    ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
-    ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
-
-    [stream_out1, stream_out2]
-  end
+  #def self.tee_stream_fork(stream)
+  #  stream_out1, stream_in1 = Misc.pipe
+  #  stream_out2, stream_in2 = Misc.pipe
+
+  #  splitter_pid = Process.fork do
+  #    Misc.purge_pipes(stream_in1, stream_in2)
+  #    stream_out1.close
+  #    stream_out2.close
+  #    begin
+  #      filename = stream.respond_to?(:filename)? stream.filename : nil
+  #      skip1 = skip2 = false
+  #      while block = stream.read(2048)
+  #        begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
+  #        begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
+  #      end
+  #      raise "Error writing in stream_in1" if skip1
+  #      raise "Error writing in stream_in2" if skip2
+  #      stream.join if stream.respond_to? :join
+  #      stream_in1.close 
+  #      stream_in2.close 
+  #    rescue Aborted
+  #      stream.abort if stream.respond_to? :abort
+  #      raise $!
+  #    rescue IOError
+  #      Log.exception $!
+  #    rescue Exception
+  #      Log.exception $!
+  #    end
+  #  end
+  #  stream.close
+  #  stream_in1.close
+  #  stream_in2.close
+
+  #  ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
+  #  ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
+
+  #  [stream_out1, stream_out2]
+  #end
 
   def self.tee_stream_thread(stream)
     stream_out1, stream_in1 = Misc.pipe
     stream_out2, stream_in2 = Misc.pipe
 
-    splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
+    splitter_thread = Thread.new(Thread.current) do |parent|
       begin
         filename = stream.respond_to?(:filename)? stream.filename : nil
         skip1 = skip2 = false
         while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1 
-          begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2 
+          begin 
+            stream_in1.write block; 
+          rescue IOError
+            Log.error("Tee stream 1 #{stream} IOError: #{$!.message}");
+            skip1 = true
+          end unless skip1 
+
+          begin 
+            stream_in2.write block
+          rescue IOError
+            Log.error("Tee stream 2 #{stream} IOError: #{$!.message}");
+            skip2 = true
+          end unless skip2 
         end
-        stream_in1.close 
-        stream_in2.close 
+        stream_in1.close unless stream_in1.closed?
+        stream_in2.close unless stream_in2.closed?
         stream.join if stream.respond_to? :join
       rescue Aborted
+        Log.error("Tee stream #{stream} Aborted");
         stream.abort if stream.respond_to? :abort
-        parent.raise $!
-      rescue IOError
-        Log.exception $!
       rescue Exception
         Log.exception $!
         parent.raise $!
@@ -207,10 +216,11 @@ module Misc
   end
 
   def self.sensiblewrite(path, content = nil, &block)
-    return if File.exists? path
+
+    return if Open.exists? path
     tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
     Misc.lock tmp_path do
-      if not File.exists? path
+      if not Open.exists? path
         FileUtils.rm_f tmp_path if File.exists? tmp_path
         begin
           case
@@ -227,10 +237,13 @@ module Misc
           else
             File.open(tmp_path, 'w') do |f|  end
           end
-          FileUtils.mv tmp_path, path
+
+          Open.mv tmp_path, path
+        rescue Aborted
+          Log.error "Aborted sensiblewrite -- #{ Log.color :blue, path }"
         rescue Exception
           Log.error "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
-          FileUtils.rm_f path if File.exists? path
+          Open.rm_f path if File.exists? path
           raise $!
         ensure
           FileUtils.rm_f tmp_path if File.exists? tmp_path

data/lib/rbbt/util/open.rb

@@ -145,29 +145,34 @@ module Open
 
   def self.get_stream_from_repo(dir, sub_path)
     repo = get_repo_from_dir(dir)
-    repo.read
-    StringIO.new repo[sub_path]
+    repo.read_and_close do
+      StringIO.new repo[sub_path]
+    end
   end
 
   def self.save_content_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
-    repo.write
-    repo[sub_path] = content
+    repo.write_and_close do
+      repo[sub_path] = content
+    end
   end
 
   def self.remove_from_repo(dir, sub_path, recursive = false)
     repo = get_repo_from_dir(dir)
-    repo.write
-    if recursive
-      repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
-    else
-      repo.outlist sub_path
+    repo.write_and_close do
+      if recursive
+        repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
+      else
+        repo.outlist sub_path
+      end
     end
   end
 
   def self.exists_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
-    repo.include? sub_path
+    repo.read_and_close do
+      repo.include? sub_path
+    end
   end
 
   def self.find_repo_dir(file)
@@ -198,7 +203,7 @@ module Open
 
   def self.file_open(file, grep, mode = 'r', invert_grep = false)
     if (dir_sub_path = find_repo_dir(file))
-      stream =  get_stream_from_repo(*dir_sub_path)
+      stream = get_stream_from_repo(*dir_sub_path)
     else
       stream =  File.open(file, mode)
     end
@@ -227,6 +232,34 @@ module Open
     end
   end
 
+  def self.mv(source, target)
+    dir_sub_path_source = find_repo_dir(source)
+    dir_sub_path_target = find_repo_dir(target)
+
+    return FileUtils.mv source, target if dir_sub_path_source.nil? and dir_sub_path_target.nil?
+
+    if dir_sub_path_source.nil?
+      save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source))
+      return nil
+    end
+
+    if dir_sub_path_target.nil?
+      Open.write(target, get_stream_from_repo(dir_sub_path_source))
+      return nil
+    end
+
+    repo_source = get_repo_from_dir(dir_sub_path_source[0])
+    repo_target = get_repo_from_dir(dir_sub_path_target[0])
+
+    repo_source.write do
+      repo_target.write do
+        repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
+      end
+    end
+
+    return nil
+  end
+
   def self.exists?(file)
     if (dir_sub_path = find_repo_dir(file))
       dir_sub_path.push file
@@ -358,6 +391,7 @@ module Open
     end
 
     io.filename = url.to_s
+
     io
   end