scout-gear 7.3.0 → 8.1.0

data/lib/scout/config.rb

@@ -148,7 +148,7 @@ module Scout::Config
   end
 
   def self.process_config(config)
-    if Misc.is_filename?(config) && File.exist?(config)
+    if Path.is_filename?(config) && File.exist?(config)
       Scout::Config.load_file(config)
     elsif Scout.etc.config_profile[config].exists?
       Scout::Config.load_file(Scout.etc.config_profile[config].find)

data/lib/scout/exceptions.rb

@@ -139,3 +139,13 @@ class SemaphoreInterrupted < TryAgain; end
 #
 #
 class ResourceNotFound < ScoutException; end
+
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/log/color.rb

@@ -10,8 +10,8 @@ module Colorize
 
   def self.colors
     @colors ||= IndiferentHash.setup(Hash[<<-EOF.split("\n").collect{|l| l.split(" ")}])
-green #00cd00 
-red #cd0000 
+green #00cd00
+red #cd0000
 yellow #ffd700
 blue #0000cd
 path blue
@@ -49,26 +49,26 @@ EOF
     when "black"
       '#fff'
     when 'green'
-      colors["green3"] 
+      colors["green3"]
     when 'red'
-      colors["red3"] 
+      colors["red3"]
     when 'yellow'
-      colors["gold1"] 
+      colors["gold1"]
     when 'blue'
-      colors["RoyalBlue"] 
+      colors["RoyalBlue"]
     else
       colors[color.to_s] || color.to_s
     end
   end
 
-  def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false) 
+  def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
     start_color = Color.new from_name(start)
     end_color = Color.new from_name(eend)
 
     if percent
       array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
     else
-      array = array.collect{|v| v.to_f } 
+      array = array.collect{|v| v.to_f }
     end
     max = array.max
     min = array.min
@@ -134,7 +134,7 @@ module Log
     attr_accessor :nocolor
   end
 
-  self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
+  self.nocolor = ENV["SCOUT_NOCOLOR"] == 'true'
 
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
@@ -152,9 +152,12 @@ module Log
     :start => cyan,
     :done => green,
     :error => red,
+    :time => cyan,
+    :task => yellow,
+    :workflow => yellow,
   })
   HIGHLIGHT = "\033[1m"
-  
+
   def self.uncolor(str)
     "" << Term::ANSIColor.uncolor(str)
   end
@@ -164,7 +167,7 @@ module Log
   end
 
   def self.color(color, str = nil, reset = false)
-    return str.dup || "" if nocolor 
+    return str.dup || "" if nocolor
 
     if (color == :integer || color == :float) && Numeric === str
       color = if str < 0
@@ -176,7 +179,7 @@ module Log
               end
     end
 
-    if color == :status 
+    if color == :status
       color = case str.to_sym
               when :done
                 :green

data/lib/scout/log/progress/report.rb

@@ -4,8 +4,7 @@ module Log
     def print(io, str)
       return if self.severity && self.severity < Log.severity
       return if Log.no_bar
-      STDERR.print str
-      Log.logfile.puts str unless Log.logfile.nil?
+      Log.log_write str
       Log::LAST.replace "progress"
     end
 
@@ -57,9 +56,12 @@ module Log
         thr = short_mean
       else
         thr = begin
-                (@ticks || 1) / (Time.now - @start) 
-              rescue
-                1
+                d = Time.now - @start
+                if d == 0
+                  1
+                else
+                  (@ticks || 1) / d
+                end
               end
       end
 
@@ -229,7 +231,7 @@ module Log
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 
-      File.rm file if file and File.exist?(file)
+      FileUtils.rm file if file and File.exist?(file)
 
       begin
         @callback.call self

data/lib/scout/log/progress/util.rb

@@ -29,7 +29,7 @@ module Log
 
     def self.new_bar(max, options = {})
       options, max = max, nil if Hash === max
-      max = options[:max] if max.nil?
+      max = options[:max] if options && max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"
@@ -74,6 +74,7 @@ module Log
       BAR_MUTEX.synchronize do
         REMOVE << bar
       end
+      cleanup_bars
       Log::LAST.replace "remove_bar" if Log::LAST == "progress"
     end
 
@@ -82,7 +83,7 @@ module Log
     end
 
     def self.with_bar(max = nil, options = {})
-      bar = new_bar(max, options)
+      bar = options.include?(:bar) ? options[:bar] : new_bar(max, options)
       begin
         error = false
         keep = false
@@ -98,69 +99,75 @@ module Log
     end
 
     def self.guess_obj_max(obj)
-    begin
-      case obj
-      when (defined? Step and Step)
-        if obj.done?
-          path = obj.path
-          path = path.find if path.respond_to? :find
-          if File.exist? path
-            CMD.cmd("wc -l '#{path}'").read.to_i 
+      begin
+        case obj
+        when (defined? Step and Step)
+          if obj.done?
+            path = obj.path
+            path = path.find if path.respond_to? :find
+            if File.exist? path
+              CMD.cmd("wc -l '#{path}'").read.to_i 
+            else
+              nil
+            end
+          else
+            nil
+          end
+        when TSV
+          obj.length
+        when Array, Hash
+          obj.size
+        when File
+          return nil if Open.gzip?(obj.filename) or Open.bgzip?(obj.filename) or Open.remote?(obj.filename)
+          CMD.cmd("wc -l '#{obj.filename}'").read.to_i
+        when Path, String
+          obj = obj.find if Path === obj
+          if File.exist? obj
+            return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+            CMD.cmd("wc -l '#{obj}'").read.to_i
           else
             nil
           end
-        else
-          nil
-        end
-      when TSV
-        obj.length
-      when Array, Hash
-        obj.size
-      when File
-        return nil if Open.gzip?(obj) or Open.bgzip?(obj)
-        CMD.cmd("wc -l '#{obj.path}'").read.to_i
-      when Path, String
-        obj = obj.find if Path === obj
-        if File.exist? obj
-          return nil if Open.gzip?(obj) or Open.bgzip?(obj)
-          CMD.cmd("wc -l '#{obj}'").read.to_i
-        else
-          nil
         end
+      rescue Interrupt
+        raise $!
+      rescue Exception
+        nil
       end
-    rescue Exception
-      Log.exception $!
-      nil
     end
-  end
 
-  def self.get_obj_bar(bar, obj)
-    case bar
-    when String
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(max, {:desc => bar}) 
-    when TrueClass
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(max, nil) 
-    when Numeric
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(bar) 
-    when Hash
-      max = Misc.process_options(bar, :max) || max
-      Log::ProgressBar.new_bar(max, bar) 
-    when Log::ProgressBar
-      bar.max ||= guess_obj_max(obj)
-      bar
-    else
-      if (defined? Step and Step === bar)
+    def self.get_obj_bar(obj, bar = nil)
+      return nil if bar.nil? || bar == false
+      case bar
+      when String
         max = guess_obj_max(obj)
-        Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
-      else
+        Log::ProgressBar.new_bar(max, {:desc => bar}) 
+      when TrueClass
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max) 
+      when Numeric
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(bar) 
+      when Hash
+        max = IndiferentHash.process_options(bar, :max) || max
+        Log::ProgressBar.new_bar(max, bar) 
+      when Log::ProgressBar
+        bar.max ||= guess_obj_max(obj)
         bar
+      else
+        if (defined? Step and Step === bar)
+          max = guess_obj_max(obj)
+          Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+        else
+          bar
+        end
       end
     end
-  end
-  end
 
+    def self.with_obj_bar(obj, bar = true, &block)
+      bar = get_obj_bar(obj, bar)
+      with_bar nil, bar: bar, &block
+    end
+  end
 end
 

data/lib/scout/log/progress.rb

@@ -8,7 +8,7 @@ module Log
 
   def self.no_bar
     @@no_bar = false unless defined?(@@no_bar)
-    @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
+    @@no_bar || ENV["SCOUT_NO_PROGRESS"] == "true"
   end
 
   class ProgressBar

data/lib/scout/log/trap.rb

@@ -0,0 +1,107 @@
+module Log
+  def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
+    sout, sin = Open.pipe
+    soute, sine = Open.pipe
+    backup_stderr = STDERR.dup
+    backup_stdout = STDOUT.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    severity_err ||= severity
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    th_loge = Thread.new do
+      while line = soute.gets
+        Log.logn "#{msge}: " + line, severity_err
+      end
+    end
+
+    begin
+      STDOUT.reopen(sin)
+      STDERR.reopen(sine)
+      yield
+    ensure
+      STDERR.reopen backup_stderr
+      STDOUT.reopen backup_stdout
+      sin.close
+      sine.close
+      th_log.join
+      th_loge.join
+      backup_stdout.close
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self.trap_stderr(msg = "STDERR", severity = 0)
+    sout, sin = Open.pipe
+    backup_stderr = STDERR.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    begin
+      STDERR.reopen(sin)
+      yield
+      sin.close
+    ensure
+      STDERR.reopen backup_stderr
+      th_log.join
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self._ignore_stderr
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
+        STDERR.reopen(f)
+        begin
+          yield
+        ensure
+          STDERR.reopen backup_stderr
+          backup_stderr.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stderr(&block)
+    _ignore_stderr &block
+  end
+
+  def self._ignore_stdout
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stdout = STDOUT.dup
+        STDOUT.reopen(f)
+        begin
+          yield
+        ensure
+          STDOUT.reopen backup_stdout
+          backup_stdout.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stdout(&block)
+    _ignore_stdout &block
+  end
+end

data/lib/scout/log.rb

@@ -1,42 +1,76 @@
 require_relative 'log/color'
 require_relative 'log/fingerprint'
 require_relative 'log/progress'
+require_relative 'log/trap'
 
 require 'io/console'
 
 module Log
+  class << self
+    attr_accessor :severity
+    attr_writer :tty_size, :logfile
+  end
+
   SEVERITY_NAMES ||= begin
-                       names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE ) 
+                       names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
                        names.each_with_index do |name,i|
-                         eval "#{ name } = #{ i }" 
+                         eval "#{ name } = #{ i }"
                        end
                        names
                      end
+  def self.default_severity
+    @@default_severity ||= begin
+                             file = File.join(ENV["HOME"], '.scout/etc/log_severity')
+                             if File.exist? file
+                               File.open(file) do |f|
+                                 SEVERITY_NAMES.index f.read.strip
+                               end
+                             else
+                               INFO
+                             end
+                           end
+    @@default_severity
+  end
 
-  class << self
-    attr_accessor :severity
-    attr_writer :tty_size, :logfile
+  case ENV['SCOUT_LOG']
+  when 'DEBUG'
+    self.severity = DEBUG
+  when 'LOW'
+    self.severity = LOW
+  when 'MEDIUM'
+    self.severity = MEDIUM
+  when 'HIGH'
+    self.severity = HIGH
+  when nil
+    self.severity = default_severity
+  else
+    self.severity = default_severity
   end
 
 
+
   def self.tty_size
-    @tty_size = begin
-                  IO.console.winsize.last
-                rescue
-                  begin
-                    `tput li`
-                  rescue
-                    80
-                  end
-                end
+    @@tty_size ||= Log.ignore_stderr do
+      begin
+        IO.console.winsize.last
+      rescue Exception
+        begin
+          res = `tput li`
+          res = nil if res == ""
+          res || ENV["TTY_SIZE"] || 80
+        rescue Exception
+          ENV["TTY_SIZE"] || 80
+        end
+      end
+    end
   end
 
 
   def self.last_caller(stack)
     line = nil
     pos ||= 0
-    while line.nil? or line =~ /scout\/log\.rb/ and stack.any? 
-      line = stack.shift 
+    while line.nil? or line =~ /scout\/log\.rb/ and stack.any?
+      line = stack.shift
     end
     line ||= caller.first
     line.gsub('`', "'")
@@ -99,9 +133,36 @@ module Log
     out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
   end
 
+  MUTEX = Mutex.new
+  def self.log_write(str)
+    MUTEX.synchronize do
+      if logfile.nil?
+        begin
+          STDERR.write str
+        rescue
+        end
+      else
+        logfile.write str
+      end
+    end
+  end
+
+  def self.log_puts(str)
+    MUTEX.synchronize do
+      if logfile.nil?
+        begin
+          STDERR.puts str
+        rescue
+        end
+      else
+        logfile.puts str
+      end
+    end
+  end
+
   LAST = "log"
   def self.logn(message = nil, severity = MEDIUM, &block)
-    return if severity < self.severity 
+    return if severity < self.severity
     message ||= block.call if block_given?
     return if message.nil?
 
@@ -109,21 +170,22 @@ module Log
 
     sev_str = severity.to_s
 
-    prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
+    if ENV["SCOUT_DEBUG_PID"] == "true"
+      prefix = time << "["  << Process.pid.to_s << "]" << color(severity) << "["  << sev_str << "]" << color(0)
+    else
+      prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
+    end
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message.to_s
 
-    if logfile.nil?
-      STDERR.write str
-    else
-      logfile.write str 
-    end
+    log_write str
+
     Log::LAST.replace "log"
     nil
   end
 
   def self.log(message = nil, severity = MEDIUM, &block)
-    return if severity < self.severity 
+    return if severity < self.severity
     message ||= block.call if block_given?
     return if message.nil?
     message = message + "\n" unless message[-1] == "\n"
@@ -187,7 +249,7 @@ module Log
   def self.exception(e)
     stack = caller
     backtrace = e.backtrace || []
-    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
       error([e.class.to_s, e.message].compact * ": " )
       error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
     else
@@ -216,9 +278,9 @@ module Log
   end
 
   def self.tsv(tsv, example = false)
-    STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
-    STDERR.puts Log.color(:blue, "=> "<< Log.fingerprint(tsv), true) 
-    STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
+    log_puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
+    log_puts Log.color(:blue, "=> "<< Log.fingerprint(tsv), true)
+    log_puts Log.color(:cyan, "=> " << tsv.summary)
     if example && ! tsv.empty?
       key = case example
             when TrueClass, :first, "first"
@@ -232,26 +294,26 @@ module Log
       values = tsv[key]
       values = [values] if tsv.type == :flat || tsv.type == :single
       if values.nil?
-        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
+        log_puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
       else
-        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
+        log_puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
         tsv.fields.zip(values).each do |field,value|
-          STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
+          log_puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
         end
       end
     end
   end
 
   def self.stack(stack)
-    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+    if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
+      log_puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
-        STDERR.puts line
+        log_puts line
       end
     else
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+      log_puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack.reverse).each do |line|
-        STDERR.puts line
+        log_puts line
       end
     end
   end
@@ -259,7 +321,7 @@ module Log
   def self.count_stack
     if ! $count_stacks
       Log.debug "Counting stacks at: " << caller.first
-      return 
+      return
     end
     $stack_counts ||= {}
     head = $count_stacks_head
@@ -280,21 +342,6 @@ module Log
     $stack_counts = {}
     res
   end
-
-  case ENV['RBBT_LOG'] 
-  when 'DEBUG' 
-    self.severity = DEBUG
-  when 'LOW' 
-    self.severity = LOW
-  when 'MEDIUM' 
-    self.severity = MEDIUM
-  when 'HIGH' 
-    self.severity = HIGH
-  when nil
-    self.severity = INFO
-  else
-    self.severity = ENV['RBBT_LOG'].to_i
-  end
 end
 
 def ppp(message)
@@ -373,3 +420,19 @@ def sss(level, &block)
   end
 end
 
+$scout_debug_log = false
+def ccc(obj=nil, file = $stdout)
+  if block_given?
+    old_scout_debug_log = $scout_debug_log
+    $scout_debug_log = 'true'
+    begin
+      yield
+    ensure
+      $scout_debug_log = old_scout_debug_log
+    end
+  else
+    Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
+  end
+end
+
+