scout-gear 7.3.0 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7273030ac34bd180620d9075ee85b26d8a9883831a5925fc911915f1edeeecf3
-  data.tar.gz: 4e445acb896844240c0024b7754cffee93af71feb62f9c4251fda70aee79121a
+  metadata.gz: 48f6bb392f3391e2816e701887bfd98617b1a68bc533c8b6de3307f25c84906d
+  data.tar.gz: 476858f63deaa7fa9fc531c7872bdaefa1d6b3ed32cdacbe41f53d1d170fcc87
 SHA512:
-  metadata.gz: '029a8eb23eda77c7e0c11ac2966e1e8ebf3eae479bbee0c76e113f5b839c655e97e0fec5025cc3c251795afcfc92ac3395a79b68c8eb99158d1cee8e2310f002'
-  data.tar.gz: 04ed4bcabfa7c41b80ba520074404bfb09c723ecdb8d02a2052f73fe86759bab7fdadff31f0f66284ed4938ebe5a1cad56fb94182b3edcc0de6325decdcb84f6
+  metadata.gz: c61501c8cc79b57b64cb8613efc8b00fb84908bf0d55b008c6a6a39b99ea6bffa53630a8e134c07a49cb7bc4145ec181df057b6ba1eebfd2d722b2aa2d2e63d0
+  data.tar.gz: 100356a04cd34eba627239668747023ee5be4455ebbee715df39dbe04e9c8dfcac7007ce310e4622cf0c7e16ee9159e6fc2d4eefcaa486c24519c7f99185facf

data/.vimproject

@@ -9,10 +9,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   find
   glob
   workflow=workflow{
-   task_old
    task
    list
    info
+   task_old
   }
  }
  lib=lib {
@@ -32,7 +32,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     filesystem.rb
     monitor.rb
     system.rb
-    helper.rb
    }
    named_array.rb
    indiferent_hash.rb
@@ -50,6 +49,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      report.rb
      util.rb
     }
+    trap.rb
    }
    tmpfile.rb
    simple_opt.rb
@@ -71,6 +71,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    open.rb
    open=open{
     lock.rb
+    lock=lock{
+     lockfile.rb
+    }
     stream.rb
     remote.rb
     util.rb
@@ -99,6 +102,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step.rb
     step=step{
      info.rb
+     status.rb
      load.rb
      file.rb
      config.rb
@@ -122,8 +126,17 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    }
    tsv.rb
    tsv=tsv{
+    util.rb
+    util=util{
+     process.rb
+     select.rb
+     process.rb
+     unzip.rb
+     reorder.rb
+    }
     parser.rb
     dumper.rb
+    transformer.rb
     persist.rb
     persist=persist{
      adapter.rb
@@ -133,12 +146,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     }
     index.rb
     traverse.rb
-    util.rb
-    util=util{
-     process.rb
-     select.rb
-     filter.rb
-    }
+    attach.rb
+    change_id.rb
     path.rb
     open.rb
    }
@@ -309,6 +318,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      workflow.rb
      workflow=workflow{
+      refactor.rb
       annotate.rb
 
       definition.rb
@@ -390,6 +400,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      tsv.rb
      tsv=tsv{
+      refactor.rb
       parser.rb
       dumper.rb
 
@@ -513,7 +524,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       test_path.rb
      }
      workflow=workflow{
-      _test_soap.rb
+      test_soap.rb
       test_doc.rb
       test_step.rb
       test_task.rb

data/VERSION

@@ -1 +1 @@
-7.3.0
+8.0.0

data/bin/scout

@@ -2,6 +2,10 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+ENV["SCOUT_NOCOLOR"] = "true" if ARGV.include? "--nocolor"
+
+ENV["SCOUT_NO_PROGRESS"] = "true" if ARGV.include? "--nobar"
+
 require 'scout-gear'
 
 class CmdStop < Exception
@@ -44,7 +48,6 @@ if dev_dir
   end
 end
 
-Log.nocolor = true if ARGV.include? "--nocolor"
 
 require 'scout/simple_opt'
 
@@ -66,11 +69,11 @@ Log.nocolor = true if options[:nocolor]
 locate = options.delete :locate_file
 
 if options[:log_file]
-  Log.logfile(options[:log_file])
+  Log.logfile(options.delete(:log_file))
 end
 
 if options[:log]
-  Log.severity = options[:log].to_i
+  Log.severity = options.delete(:log).to_i
 else
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
   if ENV["SCOUT_LOG"]

data/lib/rbbt-scout.rb

@@ -2,4 +2,5 @@ $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
 module Rbbt
   extend Resource
   self.pkgdir = 'rbbt'
+  self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end

data/lib/scout/cmd.rb

@@ -272,7 +272,7 @@ module CMD
         sout.close unless sout.closed?
 
         status = wait_thr.value
-        if not status.success? and not no_fail
+        if status && ! status.success? && ! no_fail
           if !err.empty?
             raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
           else

data/lib/scout/concurrent_stream.rb

@@ -141,7 +141,11 @@ module ConcurrentStream
       close unless closed?
     ensure
       @joined = true
-      lock.unlock if lock and lock.locked?
+      begin
+        lock.unlock if lock && lock.locked?
+      rescue 
+        Log.exception $!
+      end
       raise stream_exception if stream_exception
     end
   end
@@ -152,21 +156,17 @@ module ConcurrentStream
     name += " - file:#{filename}" if filename
     Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
 
-    @threads.each do |t| 
+    threads = @threads.dup
+    @threads.clear
+    threads.each do |t| 
       next if t == Thread.current
+      next if t["aborted"]
+      t["aborted"] = true
+      exception = exception.nil? ? Aborted.new : exception
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
-      t.raise((exception.nil? ? Aborted.new : exception))
+      t.raise(exception)
+      t.join
     end 
-
-    @threads.each do |t|
-      next if t == Thread.current
-      begin
-        t.join
-      rescue Aborted
-      rescue Exception
-        Log.debug "Thread (#{name}) exception: #{$!.message}"
-      end
-    end
   end
 
   def abort_pids
@@ -217,12 +217,11 @@ module ConcurrentStream
       begin
         super(*args)
       rescue
-        Log.exception $!
         self.abort
         self.join 
         stream_raise_exception $!
       ensure
-        self.join if self.closed? or self.eof? 
+        self.join if ! @stream_exception && (self.closed? || self.eof?)
       end
     else
       super(*args)
@@ -232,15 +231,19 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
-    rescue
-      raise stream_exception if stream_exception
-      raise $!
+    rescue Exception
+      @stream_exception ||= $!
+      raise @stream_exception
     ensure
-      begin
-        close unless closed?
-      rescue Exception
-        raise $! if ConcurrentStreamProcessFailed === $!
-      end if autojoin && ! closed? && eof?
+      if ! @stream_exception && autojoin && ! closed?
+        begin
+          done = eof?
+        rescue Exception
+          self.abort($!)
+          raise $!
+        end
+        close if done
+      end
     end
   end
 

data/lib/scout/config.rb

@@ -148,7 +148,7 @@ module Scout::Config
   end
 
   def self.process_config(config)
-    if Misc.is_filename?(config) && File.exist?(config)
+    if Path.is_filename?(config) && File.exist?(config)
       Scout::Config.load_file(config)
     elsif Scout.etc.config_profile[config].exists?
       Scout::Config.load_file(Scout.etc.config_profile[config].find)

data/lib/scout/log/color.rb

@@ -134,7 +134,7 @@ module Log
     attr_accessor :nocolor
   end
 
-  self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
+  self.nocolor = ENV["SCOUT_NOCOLOR"] == 'true'
 
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
@@ -152,6 +152,9 @@ module Log
     :start => cyan,
     :done => green,
     :error => red,
+    :time => cyan,
+    :task => yellow,
+    :workflow => yellow,
   })
   HIGHLIGHT = "\033[1m"
   

data/lib/scout/log/progress/report.rb

@@ -229,7 +229,7 @@ module Log
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 
-      File.rm file if file and File.exist?(file)
+      FileUtils.rm file if file and File.exist?(file)
 
       begin
         @callback.call self

data/lib/scout/log/progress/util.rb

@@ -29,7 +29,7 @@ module Log
 
     def self.new_bar(max, options = {})
       options, max = max, nil if Hash === max
-      max = options[:max] if max.nil?
+      max = options[:max] if options && max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"
@@ -82,7 +82,7 @@ module Log
     end
 
     def self.with_bar(max = nil, options = {})
-      bar = new_bar(max, options)
+      bar = options.include?(:bar) ? options[:bar] : new_bar(max, options)
       begin
         error = false
         keep = false
@@ -98,69 +98,73 @@ module Log
     end
 
     def self.guess_obj_max(obj)
-    begin
-      case obj
-      when (defined? Step and Step)
-        if obj.done?
-          path = obj.path
-          path = path.find if path.respond_to? :find
-          if File.exist? path
-            CMD.cmd("wc -l '#{path}'").read.to_i 
+      begin
+        case obj
+        when (defined? Step and Step)
+          if obj.done?
+            path = obj.path
+            path = path.find if path.respond_to? :find
+            if File.exist? path
+              CMD.cmd("wc -l '#{path}'").read.to_i 
+            else
+              nil
+            end
+          else
+            nil
+          end
+        when TSV
+          obj.length
+        when Array, Hash
+          obj.size
+        when File
+          return nil if Open.gzip?(obj.filename) or Open.bgzip?(obj.filename) or Open.remote?(obj.filename)
+          CMD.cmd("wc -l '#{obj.filename}'").read.to_i
+        when Path, String
+          obj = obj.find if Path === obj
+          if File.exist? obj
+            return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+            CMD.cmd("wc -l '#{obj}'").read.to_i
           else
             nil
           end
-        else
-          nil
-        end
-      when TSV
-        obj.length
-      when Array, Hash
-        obj.size
-      when File
-        return nil if Open.gzip?(obj) or Open.bgzip?(obj)
-        CMD.cmd("wc -l '#{obj.path}'").read.to_i
-      when Path, String
-        obj = obj.find if Path === obj
-        if File.exist? obj
-          return nil if Open.gzip?(obj) or Open.bgzip?(obj)
-          CMD.cmd("wc -l '#{obj}'").read.to_i
-        else
-          nil
         end
+      rescue Exception
+        nil
       end
-    rescue Exception
-      Log.exception $!
-      nil
     end
-  end
 
-  def self.get_obj_bar(bar, obj)
-    case bar
-    when String
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(max, {:desc => bar}) 
-    when TrueClass
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(max, nil) 
-    when Numeric
-      max = guess_obj_max(obj)
-      Log::ProgressBar.new_bar(bar) 
-    when Hash
-      max = Misc.process_options(bar, :max) || max
-      Log::ProgressBar.new_bar(max, bar) 
-    when Log::ProgressBar
-      bar.max ||= guess_obj_max(obj)
-      bar
-    else
-      if (defined? Step and Step === bar)
+    def self.get_obj_bar(obj, bar = nil)
+      return nil if bar.nil? || bar == false
+      case bar
+      when String
         max = guess_obj_max(obj)
-        Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
-      else
+        Log::ProgressBar.new_bar(max, {:desc => bar}) 
+      when TrueClass
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max) 
+      when Numeric
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(bar) 
+      when Hash
+        max = IndiferentHash.process_options(bar, :max) || max
+        Log::ProgressBar.new_bar(max, bar) 
+      when Log::ProgressBar
+        bar.max ||= guess_obj_max(obj)
         bar
+      else
+        if (defined? Step and Step === bar)
+          max = guess_obj_max(obj)
+          Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+        else
+          bar
+        end
       end
     end
-  end
-  end
 
+    def self.with_obj_bar(obj, bar = true, &block)
+      bar = get_obj_bar(obj, bar)
+      with_bar nil, bar: bar, &block
+    end
+  end
 end
 

data/lib/scout/log/progress.rb

@@ -8,7 +8,7 @@ module Log
 
   def self.no_bar
     @@no_bar = false unless defined?(@@no_bar)
-    @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
+    @@no_bar || ENV["SCOUT_NO_PROGRESS"] == "true"
   end
 
   class ProgressBar

data/lib/scout/log/trap.rb

@@ -0,0 +1,107 @@
+module Log
+  def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
+    sout, sin = Misc.pipe
+    soute, sine = Misc.pipe
+    backup_stderr = STDERR.dup
+    backup_stdout = STDOUT.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    severity_err ||= severity
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    th_loge = Thread.new do
+      while line = soute.gets
+        Log.logn "#{msge}: " + line, severity_err
+      end
+    end
+
+    begin
+      STDOUT.reopen(sin)
+      STDERR.reopen(sine)
+      yield
+    ensure
+      STDERR.reopen backup_stderr
+      STDOUT.reopen backup_stdout
+      sin.close
+      sine.close
+      th_log.join
+      th_loge.join
+      backup_stdout.close
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self.trap_stderr(msg = "STDERR", severity = 0)
+    sout, sin = Misc.pipe
+    backup_stderr = STDERR.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    begin
+      STDERR.reopen(sin)
+      yield
+      sin.close
+    ensure
+      STDERR.reopen backup_stderr
+      th_log.join
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self._ignore_stderr
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
+        STDERR.reopen(f)
+        begin
+          yield
+        ensure
+          STDERR.reopen backup_stderr
+          backup_stderr.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stderr(&block)
+    _ignore_stderr &block
+  end
+
+  def self._ignore_stdout
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stdout = STDOUT.dup
+        STDOUT.reopen(f)
+        begin
+          yield
+        ensure
+          STDOUT.reopen backup_stdout
+          backup_stdout.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stdout(&block)
+    _ignore_stdout &block
+  end
+end

data/lib/scout/log.rb

@@ -1,10 +1,16 @@
 require_relative 'log/color'
 require_relative 'log/fingerprint'
 require_relative 'log/progress'
+require_relative 'log/trap'
 
 require 'io/console'
 
 module Log
+  class << self
+    attr_accessor :severity
+    attr_writer :tty_size, :logfile
+  end
+
   SEVERITY_NAMES ||= begin
                        names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE ) 
                        names.each_with_index do |name,i|
@@ -12,13 +18,37 @@ module Log
                        end
                        names
                      end
+  def self.default_severity
+    @@default_severity ||= begin
+                             file = File.join(ENV["HOME"], '.scout/etc/log_severity')
+                             if File.exist? file
+                               File.open(file) do |f|
+                                 SEVERITY_NAMES.index f.read.strip
+                               end
+                             else
+                               INFO
+                             end
+                           end
+    @@default_severity
+  end
 
-  class << self
-    attr_accessor :severity
-    attr_writer :tty_size, :logfile
+  case ENV['SCOUT_LOG'] 
+  when 'DEBUG' 
+    self.severity = DEBUG
+  when 'LOW' 
+    self.severity = LOW
+  when 'MEDIUM' 
+    self.severity = MEDIUM
+  when 'HIGH' 
+    self.severity = HIGH
+  when nil
+    self.severity = default_severity
+  else
+    self.severity = default_severity
   end
 
 
+
   def self.tty_size
     @tty_size = begin
                   IO.console.winsize.last
@@ -109,7 +139,11 @@ module Log
 
     sev_str = severity.to_s
 
-    prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
+    if ENV["SCOUT_DEBUG_PID"] == "true"
+      prefix = time << "["  << Process.pid.to_s << "]" << color(severity) << "["  << sev_str << "]" << color(0)
+    else
+      prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
+    end
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message.to_s
 
@@ -187,7 +221,7 @@ module Log
   def self.exception(e)
     stack = caller
     backtrace = e.backtrace || []
-    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
       error([e.class.to_s, e.message].compact * ": " )
       error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
     else
@@ -243,7 +277,7 @@ module Log
   end
 
   def self.stack(stack)
-    if ENV["RBBT_ORIGINAL_STACK"] == 'true'
+    if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
       STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
         STDERR.puts line
@@ -280,21 +314,6 @@ module Log
     $stack_counts = {}
     res
   end
-
-  case ENV['RBBT_LOG'] 
-  when 'DEBUG' 
-    self.severity = DEBUG
-  when 'LOW' 
-    self.severity = LOW
-  when 'MEDIUM' 
-    self.severity = MEDIUM
-  when 'HIGH' 
-    self.severity = HIGH
-  when nil
-    self.severity = INFO
-  else
-    self.severity = ENV['RBBT_LOG'].to_i
-  end
 end
 
 def ppp(message)
@@ -373,3 +392,19 @@ def sss(level, &block)
   end
 end
 
+$scout_debug_log = false
+def ccc(obj=nil, file = $stdout)
+  if block_given?
+    old_scout_debug_log = $scout_debug_log
+    $scout_debug_log = 'true'
+    begin
+      yield
+    ensure
+      $scout_debug_log = old_scout_debug_log
+    end
+  else
+    Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
+  end
+end
+
+