scout-essentials 1.0.0

data/lib/scout/log/progress/report.rb

@@ -0,0 +1,244 @@
+require 'yaml'
+module Log
+  class ProgressBar
+    def print(io, str)
+      return if self.severity && self.severity < Log.severity
+      return if Log.no_bar
+      Log.log_write str
+      Log::LAST.replace "progress"
+    end
+
+    attr_accessor :history, :mean_max, :max_history
+    def thr_msg
+      if @history.nil?
+        @history ||= [[0, @start], [@ticks, Time.now] ]
+      elsif @last_ticks != @ticks
+        @history << [@ticks, Time.now]
+        max_history ||= begin
+                          max_history = case 
+                                        when @ticks > 20
+                                          count = @ticks - @last_count
+                                          count = 1 if count == 0
+                                          if @max
+                                            times = @max / count
+                                            num = times / 20
+                                            num = 2 if num < 2
+                                          else
+                                            num = 10
+                                          end
+                                          count * num
+                                        else
+                                          20
+                                        end
+                          max_history = 30 if max_history > 30
+                          max_history
+                        end
+        @history.shift if @history.length > max_history
+      end
+
+      @last_ticks = @ticks
+
+      @mean_max ||= 0
+      if @history.length > 3
+
+        sticks, stime = @history.first
+        ssticks, sstime = @history[-3]
+        lticks, ltime = @history.last
+
+
+        mean = @mean = (lticks - sticks).to_f / (ltime - stime)
+        short_mean = (lticks - ssticks).to_f / (ltime - sstime)
+
+        @mean_max = mean if mean > @mean_max
+      end
+
+      if short_mean
+        thr = short_mean
+      else
+        thr = begin
+                d = Time.now - @start
+                if d == 0
+                  1
+                else
+                  (@ticks || 1) / d
+                end
+              end
+      end
+
+      thr = 0.0000001 if thr == 0
+      
+      if mean.nil? or mean.to_i > 2
+        str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
+        #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
+      else
+        if 1.0/thr < 1
+          str = "#{ Log.color :blue, (1.0/thr).round(2).to_s } secs each"
+        elsif 1.0/thr < 2
+          str = "#{ Log.color :blue, (1.0/thr).round(1).to_s } secs each"
+        else
+          str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        end
+        #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
+      end
+
+      str
+    end
+
+
+    def eta_msg
+      percent = self.percent
+      time = Time.now
+
+      indicator = ""
+      10.times{|i|
+        if i < percent / 10 then
+          indicator << Log.color(:yellow, ".")
+        else
+          indicator << " "
+        end
+      }
+
+      indicator << " #{Log.color(:blue, percent.to_s << "%")}"
+
+      used = time - @start
+      if @mean_max and @mean_max > 0 and @mean > 0
+        eta =  (@max - @ticks) / @mean
+      else
+        eta =  (@max - @ticks) / (@ticks/used)
+      end
+
+      used = Misc.format_seconds(used) 
+      eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
+
+      #indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left - #{Log.color :yellow, ticks.to_s} of #{Log.color :yellow, @max.to_s} #{bytes ? 'bytes' : 'items'}"
+      indicator << " #{Log.color :yellow, eta} => #{Log.color :yellow, used} - #{Log.color :yellow, ticks.to_s} of #{Log.color :yellow, @max.to_s} #{bytes ? 'bytes' : 'items'}"
+
+      indicator
+    end
+
+    def report_msg
+      str = Log.color(:magenta, "·")
+      if @ticks == 0
+        if @max
+          return str << " " << Log.color(:magenta, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") <<  Log.color(:magenta, " · " << desc)
+        else
+          return str << " " << Log.color(:magenta, "waiting - PID: #{Process.pid}") <<  Log.color(:magenta, " · " << desc)
+        end
+      end
+      str << " " << thr_msg
+      if max
+        str << Log.color(:blue, " -- ") << eta_msg
+      else
+        str << Log.color(:blue, " -- ") << ticks.to_s << " #{bytes ? 'bytes' : 'items'}"
+      end
+      str <<  Log.color(:magenta, " · " << desc)
+      str
+    end
+
+    def load(info)
+      info.each do |key, value| 
+        case key.to_sym
+        when :start 
+          @start = value
+        when :last_time 
+          @last_time = value
+        when :last_count 
+          @last_count = value
+        when :last_percent 
+          @last_percent = value
+        when :desc 
+          @desc = value
+        when :ticks 
+          @ticks = value
+        when :max 
+          @max = value
+        when :mean 
+          @mean = value
+        end
+      end
+    end
+
+    def save
+      info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
+      info.delete_if{|k,v| v.nil?}
+      FileUtils.mkdir_p File.dirname(file) unless File.exist?(File.dirname(file))
+      File.write(file, info.to_yaml)
+    end
+
+    def report(io = STDERR)
+      if Log::LAST != "progress"
+        bars = BARS
+        if Log::LAST == "new_bar"
+          Log::LAST.replace "progress"
+          bar = bars.sort_by{|b| b.depth }.first
+          print(io, Log.color(:magenta ,bar.report_msg) << "\n") 
+        else
+          print(io, Log.color(:magenta, "···Progress\n"))
+          bars.sort_by{|b| b.depth }.reverse.each do |bar|
+            if SILENCED.include? bar
+              print(io, Log.color(:magenta, "·\n")) 
+            else
+              print(io, Log.color(:magenta ,bar.report_msg) << "\n") 
+            end
+          end
+        end
+      else
+        bars = BARS
+      end
+      bars << self unless BARS.include? self
+
+      print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
+      print(io, Log.up_lines(@depth) << report_msg << "\n" << Log.down_lines(@depth - 1)) 
+      @last_time = Time.now
+      @last_count = ticks
+      @last_percent = percent if max and max > 0
+      Log::LAST.replace "progress"
+      save if file
+    end
+
+    def done(io = STDERR)
+      done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
+      if @start
+        ellapsed = (Time.now - @start)
+      else
+        ellapsed = 0
+      end
+      ellapsed_str = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed_str)
+      @last_count = 0
+      @last_time = @start
+      thr = ellapsed > 0 ? (@ticks / ellapsed).to_i.to_s : 0
+      done_msg << " - " << Log.color(:blue, thr) << " per second"
+      done_msg << Log.color(:magenta, " · " << desc)
+      print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
+
+      FileUtils.rm file if file and File.exist?(file)
+
+      @callback.call self if @callback
+    end
+
+    def error(io = STDERR)
+      done_msg = Log.color(:magenta, "· ") << Log.color(:red, "error")
+      if @start
+        ellapsed = (Time.now - @start).to_i
+      else
+        ellapsed = 0
+      end
+      ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " in " << Log.color(:green, ellapsed)
+      @last_count = 0
+      @last_time = @start
+      done_msg << " - " << thr_msg
+      done_msg << Log.color(:magenta, " · " << desc)      
+      print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
+
+      FileUtils.rm file if file and File.exist?(file)
+
+      begin
+        @callback.call self
+      rescue
+        Log.debug "Callback failed for filed progress bar: #{$!.message}"
+      end if @callback
+    end
+  end
+end

data/lib/scout/log/progress/util.rb

@@ -0,0 +1,173 @@
+require_relative '../../exceptions'
+module Log
+  class ProgressBar
+    BAR_MUTEX = Mutex.new
+    BARS = []
+    REMOVE = []
+    SILENCED = []
+
+    def self.add_offset(value = 1)
+      value = 1 if TrueClass === value
+      @@offset = offset + value.to_i
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
+    def self.remove_offset(value = 1)
+      value = 1 if TrueClass === value
+      @@offset = offset - value.to_i
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
+
+    def self.offset
+      @@offset ||= 0
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
+    def self.new_bar(max, options = {})
+      options, max = max, nil if Hash === max
+      max = options[:max] if options && max.nil?
+      cleanup_bars
+      BAR_MUTEX.synchronize do
+        Log::LAST.replace "new_bar" if Log::LAST == "progress"
+        options = IndiferentHash.add_defaults options, :depth => BARS.length + Log::ProgressBar.offset
+        BARS << (bar = ProgressBar.new(max, options))
+        bar
+      end
+    end
+
+    def self.cleanup_bars
+      BAR_MUTEX.synchronize do
+        REMOVE.each do |bar|
+          index = BARS.index bar
+          if index
+            BARS.delete_at index
+            BARS.each_with_index do |bar,i|
+              bar.depth = i
+            end
+          end
+          index = SILENCED.index bar
+          if index
+            SILENCED.delete_at index
+            SILENCED.each_with_index do |bar,i|
+              bar.depth = i
+            end
+          end
+        end
+        REMOVE.clear
+        BARS.length
+      end
+    end
+
+    def self.remove_bar(bar, error = false)
+      BAR_MUTEX.synchronize do
+        return if REMOVE.include? bar
+      end
+      if error
+        bar.error if bar.respond_to? :error
+      else
+        bar.done if bar.respond_to? :done
+      end
+      BAR_MUTEX.synchronize do
+        REMOVE << bar
+      end
+      cleanup_bars
+      Log::LAST.replace "remove_bar" if Log::LAST == "progress"
+    end
+
+    def remove(error = false)
+      Log::ProgressBar.remove_bar self, error
+    end
+
+    def self.with_bar(max = nil, options = {})
+      bar = options.include?(:bar) ? options[:bar] : new_bar(max, options)
+      begin
+        error = false
+        keep = false
+        yield bar
+      rescue KeepBar
+        keep = true
+      rescue
+        error = true
+        raise $!
+      ensure
+        remove_bar(bar, error) if bar && ! keep
+      end
+    end
+
+    def self.guess_obj_max(obj)
+      begin
+        case obj
+        when (defined? Step and Step)
+          if obj.done?
+            path = obj.path
+            path = path.find if path.respond_to? :find
+            if File.exist? path
+              CMD.cmd("wc -l '#{path}'").read.to_i 
+            else
+              nil
+            end
+          else
+            nil
+          end
+        when TSV
+          obj.length
+        when Array, Hash
+          obj.size
+        when File
+          return nil if Open.gzip?(obj.filename) or Open.bgzip?(obj.filename) or Open.remote?(obj.filename)
+          CMD.cmd("wc -l '#{obj.filename}'").read.to_i
+        when Path, String
+          obj = obj.find if Path === obj
+          if File.exist? obj
+            return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+            CMD.cmd("wc -l '#{obj}'").read.to_i
+          else
+            nil
+          end
+        end
+      rescue Interrupt
+        raise $!
+      rescue Exception
+        nil
+      end
+    end
+
+    def self.get_obj_bar(obj, bar = nil)
+      return nil if bar.nil? || bar == false
+      case bar
+      when String
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max, {:desc => bar}) 
+      when TrueClass
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max) 
+      when Numeric
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(bar) 
+      when Hash
+        max = IndiferentHash.process_options(bar, :max) || max
+        Log::ProgressBar.new_bar(max, bar) 
+      when Log::ProgressBar
+        bar.max ||= guess_obj_max(obj)
+        bar
+      else
+        if (defined? Step and Step === bar)
+          max = guess_obj_max(obj)
+          Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+        else
+          bar
+        end
+      end
+    end
+
+    def self.with_obj_bar(obj, bar = true, &block)
+      bar = get_obj_bar(obj, bar)
+      with_bar nil, bar: bar, &block
+    end
+  end
+end
+

data/lib/scout/log/progress.rb

@@ -0,0 +1,106 @@
+require_relative 'progress/util'
+require_relative 'progress/report'
+module Log
+
+  def self.no_bar=(value)
+    @@no_bar = value
+  end
+
+  def self.no_bar
+    @@no_bar = false unless defined?(@@no_bar)
+    @@no_bar || ENV["SCOUT_NO_PROGRESS"] == "true"
+  end
+
+  class ProgressBar
+
+    class << self
+      attr_accessor :default_file
+      attr_accessor :default_severity
+    end
+
+    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
+
+    def initialize(max = nil, options = {})
+      depth, desc, file, bytes, frequency, process, callback, severity = 
+        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback, :severity,
+        :depth => 0, :frequency => 2, :severity => Log::ProgressBar.default_severity
+
+      max = nil if TrueClass === max
+
+      @max = max
+      @ticks = 0
+      @frequency = frequency
+      @severity = severity
+      @last_time = nil
+      @last_count = nil
+      @last_percent = nil
+      @depth = depth
+      @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
+      @file = file
+      @bytes = bytes
+      @process = process
+      @callback = callback
+    end
+
+    def percent
+      return 0 if @ticks == 0
+      return 100 if @max == 0
+      (@ticks * 100) / @max
+    end
+
+    def file
+      @file || ProgressBar.default_file
+    end
+
+    def init
+      @ticks, @bytes = 0
+      @last_time = @last_count = @last_percent = nil
+      @history, @mean_max, @max_history = nil
+      @start = @last_time = Time.now
+      @last_count = 0
+      report
+    end
+
+    def tick(step = 1)
+      return if Log.no_bar
+      @ticks += step
+
+      time = Time.now
+      if @last_time.nil?
+        @last_time = time
+        @last_count = @ticks
+        @start = time
+        return
+      end
+
+      diff = time - @last_time
+      report and return if diff >= @frequency
+      return unless max and max > 0
+
+      percent = self.percent
+      if @last_percent.nil?
+        @last_percent = percent
+        return
+      end
+      report && return if percent > @last_percent and diff > 0.3
+    end
+
+    def pos(pos)
+      step = pos - (@ticks || 0)
+      tick(step)
+    end
+
+    def process(elem)
+      case res = @process.call(elem)
+      when FalseClass
+        nil
+      when TrueClass
+        tick
+      when Integer
+        pos(res)
+      when Float
+        pos(res * max)
+      end
+    end
+  end
+end

data/lib/scout/log/trap.rb

@@ -0,0 +1,107 @@
+module Log
+  def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
+    sout, sin = Open.pipe
+    soute, sine = Open.pipe
+    backup_stderr = STDERR.dup
+    backup_stdout = STDOUT.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    severity_err ||= severity
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    th_loge = Thread.new do
+      while line = soute.gets
+        Log.logn "#{msge}: " + line, severity_err
+      end
+    end
+
+    begin
+      STDOUT.reopen(sin)
+      STDERR.reopen(sine)
+      yield
+    ensure
+      STDERR.reopen backup_stderr
+      STDOUT.reopen backup_stdout
+      sin.close
+      sine.close
+      th_log.join
+      th_loge.join
+      backup_stdout.close
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self.trap_stderr(msg = "STDERR", severity = 0)
+    sout, sin = Open.pipe
+    backup_stderr = STDERR.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    begin
+      STDERR.reopen(sin)
+      yield
+      sin.close
+    ensure
+      STDERR.reopen backup_stderr
+      th_log.join
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self._ignore_stderr
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
+        STDERR.reopen(f)
+        begin
+          yield
+        ensure
+          STDERR.reopen backup_stderr
+          backup_stderr.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stderr(&block)
+    _ignore_stderr &block
+  end
+
+  def self._ignore_stdout
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stdout = STDOUT.dup
+        STDOUT.reopen(f)
+        begin
+          yield
+        ensure
+          STDOUT.reopen backup_stdout
+          backup_stdout.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stdout(&block)
+    _ignore_stdout &block
+  end
+end