scout-gear 7.2.0 → 8.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de499ccbdffd888298324a025b253f0dd0e359d279d6abe8331acd91fd00cc57
-  data.tar.gz: 9643053080da23a98209ebefbcd3b3043b7031374a405ffed112460f1ddcf411
+  metadata.gz: 48f6bb392f3391e2816e701887bfd98617b1a68bc533c8b6de3307f25c84906d
+  data.tar.gz: 476858f63deaa7fa9fc531c7872bdaefa1d6b3ed32cdacbe41f53d1d170fcc87
 SHA512:
-  metadata.gz: 73afbf6aeb4032380fc45e2f60e2e9335d763c6837ef3bb8247c83bfaf2cd962a6fdfba767788b988e00f685d0e9bc19927a4e794fc092296cc088f23d4565ee
-  data.tar.gz: 309e86afe466c649800fd212fd22e9e183a5e92ba558d88ec0258690a15437952ead3e968c3693b89c036dc2fa9e27f4d129f6be6c001031dc38ab071361ba99
+  metadata.gz: c61501c8cc79b57b64cb8613efc8b00fb84908bf0d55b008c6a6a39b99ea6bffa53630a8e134c07a49cb7bc4145ec181df057b6ba1eebfd2d722b2aa2d2e63d0
+  data.tar.gz: 100356a04cd34eba627239668747023ee5be4455ebbee715df39dbe04e9c8dfcac7007ce310e4622cf0c7e16ee9159e6fc2d4eefcaa486c24519c7f99185facf

data/.vimproject

@@ -9,10 +9,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   find
   glob
   workflow=workflow{
-   task_old
    task
    list
    info
+   task_old
   }
  }
  lib=lib {
@@ -49,6 +49,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      report.rb
      util.rb
     }
+    trap.rb
    }
    tmpfile.rb
    simple_opt.rb
@@ -70,6 +71,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    open.rb
    open=open{
     lock.rb
+    lock=lock{
+     lockfile.rb
+    }
     stream.rb
     remote.rb
     util.rb
@@ -98,11 +102,13 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step.rb
     step=step{
      info.rb
+     status.rb
      load.rb
      file.rb
      config.rb
      dependencies.rb
      provenance.rb
+     progress.rb
     }
     task.rb
     task=task{
@@ -120,29 +126,66 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    }
    tsv.rb
    tsv=tsv{
+    util.rb
+    util=util{
+     process.rb
+     select.rb
+     process.rb
+     unzip.rb
+     reorder.rb
+    }
     parser.rb
     dumper.rb
+    transformer.rb
     persist.rb
-    index.rb
     persist=persist{
      adapter.rb
+     serialize.rb
      tokyocabinet.rb
+     fix_width_table.rb
     }
+    index.rb
     traverse.rb
-    util.rb
+    attach.rb
+    change_id.rb
     path.rb
+    open.rb
    }
   }
  }
  test=test {
   test_helper.rb
+  test_scout-gear.rb
   scout=scout{
+   test_cmd.rb
+   test_concurrent_stream.rb
+   test_config.rb
    test_indiferent_hash.rb
+   test_log.rb
+   test_meta_extension.rb
+   test_misc.rb
+   test_named_array.rb
+   test_open.rb
+   test_path.rb
+   test_persist.rb
+   test_resource.rb
+   test_semaphore.rb
+   test_tmpfile.rb
+   test_tsv.rb
+   test_work_queue.rb
+   test_workflow.rb
+   workflow=workflow{
+    test_definition.rb
+    test_documentation.rb
+    test_step.rb
+    test_task.rb
+    test_usage.rb
+    test_util.rb
+    }
    indiferent_hash=indiferent_hash{
+    test_case_insensitive.rb
     test_options.rb
    }
-   test_log.rb
-   test_tmpfile.rb
   }
  }
  share=share{
@@ -275,6 +318,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      workflow.rb
      workflow=workflow{
+      refactor.rb
       annotate.rb
 
       definition.rb
@@ -356,6 +400,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      tsv.rb
      tsv=tsv{
+      refactor.rb
       parser.rb
       dumper.rb
 
@@ -479,7 +524,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       test_path.rb
      }
      workflow=workflow{
-      _test_soap.rb
+      test_soap.rb
       test_doc.rb
       test_step.rb
       test_task.rb

data/VERSION

@@ -1 +1 @@
-7.2.0
+8.0.0

data/bin/scout

@@ -2,6 +2,10 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+ENV["SCOUT_NOCOLOR"] = "true" if ARGV.include? "--nocolor"
+
+ENV["SCOUT_NO_PROGRESS"] = "true" if ARGV.include? "--nobar"
+
 require 'scout-gear'
 
 class CmdStop < Exception
@@ -44,7 +48,6 @@ if dev_dir
   end
 end
 
-Log.nocolor = true if ARGV.include? "--nocolor"
 
 require 'scout/simple_opt'
 
@@ -66,11 +69,11 @@ Log.nocolor = true if options[:nocolor]
 locate = options.delete :locate_file
 
 if options[:log_file]
-  Log.logfile(options[:log_file])
+  Log.logfile(options.delete(:log_file))
 end
 
 if options[:log]
-  Log.severity = options[:log].to_i
+  Log.severity = options.delete(:log).to_i
 else
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
   if ENV["SCOUT_LOG"]

data/lib/rbbt-scout.rb

@@ -2,4 +2,5 @@ $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
 module Rbbt
   extend Resource
   self.pkgdir = 'rbbt'
+  self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end

data/lib/scout/cmd.rb

@@ -272,7 +272,7 @@ module CMD
         sout.close unless sout.closed?
 
         status = wait_thr.value
-        if not status.success? and not no_fail
+        if status && ! status.success? && ! no_fail
           if !err.empty?
             raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
           else

data/lib/scout/concurrent_stream.rb

@@ -9,11 +9,11 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
     
-    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair, :next
+    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -54,7 +54,7 @@ module ConcurrentStream
 
     stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
 
-    stream.lockfile = lockfile unless lockfile.nil?
+    stream.lock = lock unless lock.nil?
 
     stream.aborted = false
 
@@ -62,7 +62,7 @@ module ConcurrentStream
   end
 
   def annotate(stream)
-    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
     stream
   end
 
@@ -136,11 +136,16 @@ module ConcurrentStream
     begin
       join_threads
       join_pids
+      raise stream_exception if stream_exception
       join_callback
       close unless closed?
     ensure
       @joined = true
-      lockfile.unlock if lockfile and lockfile.locked?
+      begin
+        lock.unlock if lock && lock.locked?
+      rescue 
+        Log.exception $!
+      end
       raise stream_exception if stream_exception
     end
   end
@@ -151,21 +156,17 @@ module ConcurrentStream
     name += " - file:#{filename}" if filename
     Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
 
-    @threads.each do |t| 
+    threads = @threads.dup
+    @threads.clear
+    threads.each do |t| 
       next if t == Thread.current
+      next if t["aborted"]
+      t["aborted"] = true
+      exception = exception.nil? ? Aborted.new : exception
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
-      t.raise((exception.nil? ? Aborted.new : exception))
+      t.raise(exception)
+      t.join
     end 
-
-    @threads.each do |t|
-      next if t == Thread.current
-      begin
-        t.join unless t == Thread.current
-      rescue Aborted
-      rescue Exception
-        Log.debug "Thread (#{name}) exception: #{$!.message}"
-      end
-    end
   end
 
   def abort_pids
@@ -205,8 +206,8 @@ module ConcurrentStream
     ensure
       close unless closed?
 
-      if lockfile and lockfile.locked?
-        lockfile.unlock 
+      if lock and lock.locked?
+        lock.unlock 
       end
     end
   end
@@ -216,12 +217,11 @@ module ConcurrentStream
       begin
         super(*args)
       rescue
-        Log.exception $!
         self.abort
         self.join 
         stream_raise_exception $!
       ensure
-        self.join if self.closed? or self.eof? 
+        self.join if ! @stream_exception && (self.closed? || self.eof?)
       end
     else
       super(*args)
@@ -231,15 +231,19 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
-    rescue
-      raise stream_exception if stream_exception
-      raise $!
+    rescue Exception
+      @stream_exception ||= $!
+      raise @stream_exception
     ensure
-      begin
-        close unless closed?
-      rescue Exception
-        raise $! if ConcurrentStreamProcessFailed === $!
-      end if autojoin && ! closed? && eof?
+      if ! @stream_exception && autojoin && ! closed?
+        begin
+          done = eof?
+        rescue Exception
+          self.abort($!)
+          raise $!
+        end
+        close if done
+      end
     end
   end
 

data/lib/scout/config.rb

@@ -148,7 +148,7 @@ module Scout::Config
   end
 
   def self.process_config(config)
-    if Misc.is_filename?(config) && File.exist?(config)
+    if Path.is_filename?(config) && File.exist?(config)
       Scout::Config.load_file(config)
     elsif Scout.etc.config_profile[config].exists?
       Scout::Config.load_file(Scout.etc.config_profile[config].find)

data/lib/scout/exceptions.rb

@@ -92,6 +92,7 @@ class WorkerException < ScoutException
   end
 end
 
+class SemaphoreInterrupted < TryAgain; end
 
 #class OpenGzipError < StandardError; end
 #

data/lib/scout/log/color.rb

@@ -2,7 +2,6 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
-require 'colorist'
 
 module Colorize
   def self.colors=(colors)
@@ -135,7 +134,7 @@ module Log
     attr_accessor :nocolor
   end
 
-  self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
+  self.nocolor = ENV["SCOUT_NOCOLOR"] == 'true'
 
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
@@ -153,6 +152,9 @@ module Log
     :start => cyan,
     :done => green,
     :error => red,
+    :time => cyan,
+    :task => yellow,
+    :workflow => yellow,
   })
   HIGHLIGHT = "\033[1m"
   

data/lib/scout/log/progress/report.rb

@@ -229,7 +229,7 @@ module Log
       done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 
-      File.rm file if file and File.exist?(file)
+      FileUtils.rm file if file and File.exist?(file)
 
       begin
         @callback.call self

data/lib/scout/log/progress/util.rb

@@ -29,7 +29,7 @@ module Log
 
     def self.new_bar(max, options = {})
       options, max = max, nil if Hash === max
-      max = options[:max] if max.nil?
+      max = options[:max] if options && max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"
@@ -82,7 +82,7 @@ module Log
     end
 
     def self.with_bar(max = nil, options = {})
-      bar = new_bar(max, options)
+      bar = options.include?(:bar) ? options[:bar] : new_bar(max, options)
       begin
         error = false
         keep = false
@@ -96,6 +96,75 @@ module Log
         remove_bar(bar, error) if bar && ! keep
       end
     end
+
+    def self.guess_obj_max(obj)
+      begin
+        case obj
+        when (defined? Step and Step)
+          if obj.done?
+            path = obj.path
+            path = path.find if path.respond_to? :find
+            if File.exist? path
+              CMD.cmd("wc -l '#{path}'").read.to_i 
+            else
+              nil
+            end
+          else
+            nil
+          end
+        when TSV
+          obj.length
+        when Array, Hash
+          obj.size
+        when File
+          return nil if Open.gzip?(obj.filename) or Open.bgzip?(obj.filename) or Open.remote?(obj.filename)
+          CMD.cmd("wc -l '#{obj.filename}'").read.to_i
+        when Path, String
+          obj = obj.find if Path === obj
+          if File.exist? obj
+            return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+            CMD.cmd("wc -l '#{obj}'").read.to_i
+          else
+            nil
+          end
+        end
+      rescue Exception
+        nil
+      end
+    end
+
+    def self.get_obj_bar(obj, bar = nil)
+      return nil if bar.nil? || bar == false
+      case bar
+      when String
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max, {:desc => bar}) 
+      when TrueClass
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max) 
+      when Numeric
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(bar) 
+      when Hash
+        max = IndiferentHash.process_options(bar, :max) || max
+        Log::ProgressBar.new_bar(max, bar) 
+      when Log::ProgressBar
+        bar.max ||= guess_obj_max(obj)
+        bar
+      else
+        if (defined? Step and Step === bar)
+          max = guess_obj_max(obj)
+          Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+        else
+          bar
+        end
+      end
+    end
+
+    def self.with_obj_bar(obj, bar = true, &block)
+      bar = get_obj_bar(obj, bar)
+      with_bar nil, bar: bar, &block
+    end
   end
 end
 

data/lib/scout/log/progress.rb

@@ -8,7 +8,7 @@ module Log
 
   def self.no_bar
     @@no_bar = false unless defined?(@@no_bar)
-    @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
+    @@no_bar || ENV["SCOUT_NO_PROGRESS"] == "true"
   end
 
   class ProgressBar

data/lib/scout/log/trap.rb

@@ -0,0 +1,107 @@
+module Log
+  def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
+    sout, sin = Misc.pipe
+    soute, sine = Misc.pipe
+    backup_stderr = STDERR.dup
+    backup_stdout = STDOUT.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    severity_err ||= severity
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    th_loge = Thread.new do
+      while line = soute.gets
+        Log.logn "#{msge}: " + line, severity_err
+      end
+    end
+
+    begin
+      STDOUT.reopen(sin)
+      STDERR.reopen(sine)
+      yield
+    ensure
+      STDERR.reopen backup_stderr
+      STDOUT.reopen backup_stdout
+      sin.close
+      sine.close
+      th_log.join
+      th_loge.join
+      backup_stdout.close
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self.trap_stderr(msg = "STDERR", severity = 0)
+    sout, sin = Misc.pipe
+    backup_stderr = STDERR.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    begin
+      STDERR.reopen(sin)
+      yield
+      sin.close
+    ensure
+      STDERR.reopen backup_stderr
+      th_log.join
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self._ignore_stderr
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
+        STDERR.reopen(f)
+        begin
+          yield
+        ensure
+          STDERR.reopen backup_stderr
+          backup_stderr.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stderr(&block)
+    _ignore_stderr &block
+  end
+
+  def self._ignore_stdout
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stdout = STDOUT.dup
+        STDOUT.reopen(f)
+        begin
+          yield
+        ensure
+          STDOUT.reopen backup_stdout
+          backup_stdout.close
+        end
+      end
+    rescue Errno::ENOENT
+      yield
+    end
+  end
+
+
+  def self.ignore_stdout(&block)
+    _ignore_stdout &block
+  end
+end