scout-gear 6.0.0 → 7.2.0

data/VERSION

@@ -1 +1 @@
-6.0.0
+7.2.0

data/bin/scout

@@ -37,6 +37,11 @@ if dev_dir
       $LOAD_PATH.unshift f
     end
   end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
 end
 
 Log.nocolor = true if ARGV.include? "--nocolor"
@@ -112,7 +117,6 @@ def commands(prev)
 end
 
 def scout_usage(prev = nil)
-  puts
   puts SOPT.doc
 
   if prev

data/lib/rbbt-scout.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
+module Rbbt
+  extend Resource
+  self.pkgdir = 'rbbt'
+end

data/lib/scout/concurrent_stream.rb

@@ -9,11 +9,11 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
     
-    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
+    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -24,6 +24,7 @@ module ConcurrentStream
     stream.no_fail = no_fail unless no_fail.nil?
     stream.std_err = ""
 
+    stream.next = next_stream unless next_stream.nil?
     stream.pair = pair unless pair.nil?
 
     callback = block if block_given?
@@ -230,6 +231,9 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
+    rescue
+      raise stream_exception if stream_exception
+      raise $!
     ensure
       begin
         close unless closed?

data/lib/scout/config.rb

@@ -0,0 +1,168 @@
+require_relative 'path'
+require_relative 'resource'
+require_relative 'resource/scout'
+
+module Scout::Config
+
+  CACHE = IndiferentHash.setup({})
+
+  GOT_KEYS=[]
+
+  def self.add_entry(key, value, tokens)
+    tokens = [tokens] unless Array === tokens
+    tokens << "key:#{key}" unless tokens.include?("key:#{key}")
+    CACHE[key.to_s] ||= [] 
+    CACHE[key.to_s] << [tokens, value]
+  end
+
+  def self.load_file(file)
+    Log.debug "Loading config file: #{ file }"
+    TSV.traverse file, :type => :array do |line|
+      next if line =~ /^#/
+      key, value, *tokens = line.strip.split(/\s/)
+
+      self.add_entry(key, value, tokens) if key
+    end
+  end
+
+  def self.load_config
+    Scout.etc.config.find_all.reverse.each do |file|
+      self.load_file(file)
+    end
+  end
+
+  def self.set(values, *tokens)
+    if not Hash === values
+      values = {values => tokens.shift}
+    end
+
+    values.each do |key,value|
+      add_entry key, value, tokens
+    end
+  end
+
+  def self.token_priority(token)
+    token, _sep, priority = token.to_s.partition("::")
+
+    if priority.nil? || priority.empty?
+      type, _sep, rest = token.partition(":")
+      priority = case type
+                 when "workflow"
+                   4
+                 when "task"
+                   3
+                 when "file"
+                   2
+                 when "line"
+                   1
+                 when "key"
+                   20
+                 else
+                   10
+                 end
+    else
+      priority = priority.to_i
+    end
+
+    [token, priority]
+  end
+
+  def self.match(entries, give_token)
+    priorities = {}
+    entries.each do |tokens, value|
+      best_prio = nil
+      tokens = [tokens] unless Array === tokens
+      tokens.each do |tok|
+        tok, prio = token_priority tok
+        next unless tok == give_token
+
+        best_prio = prio if best_prio.nil? or best_prio > prio
+        next if prio > best_prio
+
+        priorities[prio] ||= []
+        priorities[prio].unshift value
+      end
+    end if entries
+    priorities
+  end
+
+  # For equal priorities the matching prioritizes tokens ealier in the list
+  def self.get(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    default = options.nil? ? nil : options[:default]
+
+    tokens = ["key:" + key] if tokens.empty?
+
+    tokens = tokens.flatten
+    file, _sep, line = caller.reject{|l| 
+      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+        l =~ /rbbt\/resource\/path\.rb/ or
+        l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /accessor\.rb/ or
+        l =~ /progress-monitor\.rb/ 
+    }.first.partition(":")
+
+    File.expand_path(file)
+
+    tokens << ("file:" << file)
+    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
+
+    entries = CACHE[key.to_s]
+    priorities = {}
+    tokens.each do |token|
+      token_prio = match entries, token.to_s
+      token_prio.each do |prio, values|
+        priorities[prio] ||= []
+        priorities[prio].concat(values)
+      end
+    end
+
+    value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
+    value = false if value == 'false'
+
+    Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
+    GOT_KEYS << [key, value, tokens]
+
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    elsif value == 'nil'
+      nil
+    else
+      value
+    end
+  end
+
+  def self.with_config
+      saved_config = {}
+      CACHE.each do |k,v|
+        saved_config[k] = v.dup
+      end
+      saved_got_keys = GOT_KEYS.dup
+    begin
+      yield
+    ensure
+      CACHE.replace(saved_config)
+      GOT_KEYS.replace(saved_got_keys)
+    end
+  end
+
+  def self.process_config(config)
+    if Misc.is_filename?(config) && File.exist?(config)
+      Scout::Config.load_file(config)
+    elsif Scout.etc.config_profile[config].exists?
+      Scout::Config.load_file(Scout.etc.config_profile[config].find)
+    else
+      key, value, *tokens = config.split(/\s/)
+      tokens = tokens.collect do |tok|
+        tok, _sep, prio = tok.partition("::")
+        prio = "0" if prio.nil? or prio.empty?
+        [tok, prio] * "::"
+      end
+      Scout::Config.set({key => value}, *tokens)
+    end
+  end
+
+
+  self.load_config
+end

data/lib/scout/exceptions.rb

@@ -84,6 +84,14 @@ class DoneProcessing < Exception
   end
 end
 
+class WorkerException < ScoutException
+  attr_accessor :worker_exception, :pid
+  def initialize(worker_exception, pid)
+    @worker_exception = worker_exception
+    @pid = pid
+  end
+end
+
 
 #class OpenGzipError < StandardError; end
 #
@@ -129,3 +137,4 @@ end
 #
 #
 #
+class ResourceNotFound < ScoutException; end

data/lib/scout/indiferent_hash/options.rb

@@ -29,6 +29,7 @@ module IndiferentHash
 
   def self.pull_keys(hash, prefix)
     new = {}
+    prefix = prefix.to_s
     hash.keys.each do |key|
       if key.to_s =~ /#{ prefix }_(.*)/
         case

data/lib/scout/indiferent_hash.rb

@@ -58,9 +58,11 @@ module IndiferentHash
   def delete(key)
     case key
     when Symbol, Module
-      super(key) || super(key.to_s)
+      v = super(key) 
+      v.nil? ? super(key.to_s) : v
     when String
-      super(key) || super(key.to_sym)
+      v = super(key)
+      v.nil? ? super(key.to_sym) : v
     else
       super(key)
     end

data/lib/scout/log/color.rb

@@ -143,13 +143,14 @@ module Log
   CONCEPT_COLORS = IndiferentHash.setup({
     :title => magenta,
     :path => blue,
-    :input => blue,
+    :input => cyan,
     :value => green,
     :integer => green,
     :negative => red,
     :float => green,
     :waiting => yellow,
-    :started => blue,
+    :started => cyan,
+    :start => cyan,
     :done => green,
     :error => red,
   })
@@ -166,10 +167,38 @@ module Log
   def self.color(color, str = nil, reset = false)
     return str.dup || "" if nocolor 
 
+    if (color == :integer || color == :float) && Numeric === str
+      color = if str < 0
+                :red
+              elsif str > 1
+                :cyan
+              else
+                :green
+              end
+    end
+
+    if color == :status 
+      color = case str.to_sym
+              when :done
+                :green
+              when :error, :aborted
+                :red
+              when :waiting, :queued
+                :yellow
+              when :started, :streamming
+                :cyan
+              when :start
+                :title
+              else
+                :cyan
+              end
+    end
+
     color = SEVERITY_COLOR[color] if Integer === color
     color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
     color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
 
+    str = str.to_s unless str.nil?
     return str if Symbol === color
     color_str = reset ? Term::ANSIColor.reset : ""
     color_str << color

data/lib/scout/log/progress/report.rb

@@ -2,6 +2,7 @@ require 'yaml'
 module Log
   class ProgressBar
     def print(io, str)
+      return if self.severity && self.severity < Log.severity
       return if Log.no_bar
       STDERR.print str
       Log.logfile.puts str unless Log.logfile.nil?

data/lib/scout/log/progress/util.rb

@@ -28,6 +28,8 @@ module Log
     end
 
     def self.new_bar(max, options = {})
+      options, max = max, nil if Hash === max
+      max = options[:max] if max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"
@@ -79,7 +81,7 @@ module Log
       Log::ProgressBar.remove_bar self, error
     end
 
-    def self.with_bar(max, options = {})
+    def self.with_bar(max = nil, options = {})
       bar = new_bar(max, options)
       begin
         error = false

data/lib/scout/log/progress.rb

@@ -15,18 +15,22 @@ module Log
 
     class << self
       attr_accessor :default_file
+      attr_accessor :default_severity
     end
 
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
 
     def initialize(max = nil, options = {})
-      depth, desc, file, bytes, frequency, process, callback = 
-        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
-        :depth => 0, :frequency => 2
+      depth, desc, file, bytes, frequency, process, callback, severity = 
+        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback, :severity,
+        :depth => 0, :frequency => 2, :severity => Log::ProgressBar.default_severity
+
+      max = nil if TrueClass === max
 
       @max = max
       @ticks = 0
       @frequency = frequency
+      @severity = severity
       @last_time = nil
       @last_count = nil
       @last_percent = nil

data/lib/scout/log.rb

@@ -186,11 +186,12 @@ module Log
 
   def self.exception(e)
     stack = caller
+    backtrace = e.backtrace || []
     if ENV["RBBT_ORIGINAL_STACK"] == 'true'
       error([e.class.to_s, e.message].compact * ": " )
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
     else
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace.reverse)*"\n")
       error([e.class.to_s, e.message].compact * ": " )
     end
   end
@@ -206,7 +207,11 @@ module Log
       line = line.sub('`',"'")
       color = :green if line =~ /workflow/
       color = :blue if line =~ /scout-/
-      Log.color color, line
+      if color
+        Log.color color, line
+      else
+        line
+      end
     end unless stack.nil?
   end
 

data/lib/scout/misc/digest.rb

@@ -41,9 +41,7 @@ module Misc
 
   def self.digest(obj)
     str = Misc.digest_str(obj)
-    hash = Digest::MD5.hexdigest(str)
-    Log.debug "Digest #{hash} - #{str}"
-    hash
+    Digest::MD5.hexdigest(str)
   end
 
   def self.file_md5(file)

data/lib/scout/misc/monitor.rb

@@ -1,4 +1,7 @@
 module Misc
+  def self.pid_alive?(pid)
+    !! Process.kill(0, pid) rescue false
+  end
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil

data/lib/scout/misc/system.rb

@@ -0,0 +1,15 @@
+module Misc
+  def self.env_add(var, value, sep = ":", prepend = true)
+    if ENV[var].nil?
+      ENV[var] = value
+    elsif ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
+      return
+    else
+      if prepend
+        ENV[var] = value + sep + ENV[var]
+      else
+        ENV[var] += sep + value
+      end
+    end
+  end
+end

data/lib/scout/misc.rb

@@ -3,6 +3,7 @@ require_relative 'misc/insist'
 require_relative 'misc/digest'
 require_relative 'misc/filesystem'
 require_relative 'misc/monitor'
+require_relative 'misc/system'
 
 module Misc
 end

data/lib/scout/named_array.rb

@@ -0,0 +1,68 @@
+require_relative 'meta_extension'
+module NamedArray
+  extend MetaExtension
+  extension_attr :fields
+
+  def self.identify_name(names, selected)
+    res = (Array === selected ? selected : [selected]).collect do |field|
+      case field
+      when nil
+        0
+      when Integer
+        field
+      when Symbol
+        field == :key ? field : identify_name(names, field.to_s)
+      when (names.nil? and String)
+        if field =~ /^\d+$/
+          identify_field(key_field, fields, field.to_i)
+        else
+          raise "No name information available and specified name not numeric: #{ field }"
+        end
+      when Symbol
+        names.index{|f| f.to_s == field.to_s }
+      when String
+        pos = names.index{|f| f.to_s == field }
+        next pos if pos
+        if field =~ /^\d+$/
+          next identify_names(names, field.to_i)
+        end
+        pos = names.index{|name| name.start_with?(field) }
+        next pos if pos
+        nil
+      else
+        raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
+      end
+    end
+
+    Array === selected ? res : res.first
+  end
+
+  def positions(fields)
+    if Array ==  fields
+      fields.collect{|field|
+        NamedArray.identify_name(@fields, field)
+      }
+    else
+      NamedArray.identify_name(@fields, fields)
+    end
+  end
+
+  def [](key)
+    pos = NamedArray.identify_name(@fields, key)
+    super(pos)
+  end
+
+  def concat(other)
+    super(other)
+    self.fields.concat(other.fields) if NamedArray === other
+    self
+  end
+
+  def to_hash
+    hash = {}
+    self.fields.zip(self) do |field,value|
+      hash[field] = value
+    end
+    IndiferentHash.setup hash
+  end
+end