scout-gear 7.1.0 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d00b2f7d1a5156c6259729b245722979b34132ca3a66922110662fadfc56b65
-  data.tar.gz: 057a511570907a68d6ee7d962705bfe3cc92325c1fc026b45c5ff0dfb0634f96
+  metadata.gz: 7273030ac34bd180620d9075ee85b26d8a9883831a5925fc911915f1edeeecf3
+  data.tar.gz: 4e445acb896844240c0024b7754cffee93af71feb62f9c4251fda70aee79121a
 SHA512:
-  metadata.gz: 8ca8280c5f4ac63ca98cc8c2388700e29752ec37b008d3f9b54a7bab13569af24fc94afbbfc60d9e295ca81c9a2e5a64bdab83e6cab7c9dd6d531013d72f6954
-  data.tar.gz: c056280d255116ed9a3c74ee8c1b4c367f30abf6ec60d278180534afc1728d104f89ac247fa0b1d7fb751692b869e2cf73a6a8ac8b6bef8f07e411a59e476409
+  metadata.gz: '029a8eb23eda77c7e0c11ac2966e1e8ebf3eae479bbee0c76e113f5b839c655e97e0fec5025cc3c251795afcfc92ac3395a79b68c8eb99158d1cee8e2310f002'
+  data.tar.gz: 04ed4bcabfa7c41b80ba520074404bfb09c723ecdb8d02a2052f73fe86759bab7fdadff31f0f66284ed4938ebe5a1cad56fb94182b3edcc0de6325decdcb84f6

data/.vimproject

@@ -18,6 +18,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  lib=lib {
   scout-gear.rb
   workflow-scout.rb
+  rbbt-scout.rb
   scout.rb
   scout=scout{
    meta_extension.rb
@@ -30,7 +31,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     digest.rb
     filesystem.rb
     monitor.rb
+    system.rb
+    helper.rb
    }
+   named_array.rb
    indiferent_hash.rb
    indiferent_hash=indiferent_hash{
     case_insensitive.rb
@@ -77,10 +81,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     produce=produce{
      rake.rb
     }
+    software.rb
     scout.rb
     util.rb
     path.rb
    }
+   config.rb
    persist.rb
    persist=persist{
     serialize.rb
@@ -94,6 +100,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step=step{
      info.rb
      load.rb
+     file.rb
+     config.rb
+     dependencies.rb
+     provenance.rb
+     progress.rb
     }
     task.rb
     task=task{
@@ -112,21 +123,70 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    tsv.rb
    tsv=tsv{
     parser.rb
+    dumper.rb
+    persist.rb
+    persist=persist{
+     adapter.rb
+     serialize.rb
+     tokyocabinet.rb
+     fix_width_table.rb
+    }
+    index.rb
+    traverse.rb
+    util.rb
+    util=util{
+     process.rb
+     select.rb
+     filter.rb
+    }
+    path.rb
+    open.rb
    }
   }
  }
  test=test {
   test_helper.rb
+  test_scout-gear.rb
   scout=scout{
+   test_cmd.rb
+   test_concurrent_stream.rb
+   test_config.rb
    test_indiferent_hash.rb
+   test_log.rb
+   test_meta_extension.rb
+   test_misc.rb
+   test_named_array.rb
+   test_open.rb
+   test_path.rb
+   test_persist.rb
+   test_resource.rb
+   test_semaphore.rb
+   test_tmpfile.rb
+   test_tsv.rb
+   test_work_queue.rb
+   test_workflow.rb
+   workflow=workflow{
+    test_definition.rb
+    test_documentation.rb
+    test_step.rb
+    test_task.rb
+    test_usage.rb
+    test_util.rb
+    }
    indiferent_hash=indiferent_hash{
+    test_case_insensitive.rb
     test_options.rb
    }
-   test_log.rb
-   test_tmpfile.rb
+  }
+ }
+ share=share{
+  software=software{
+   install_helpers
   }
  }
 
+
+
  modules=modules{
   rbbt-util=rbbt-util filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.txt *.conf LICENSE" {
    Migration-1.9.txt
@@ -386,6 +446,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      resource.rb
      resource=resource{
       path.rb
+      path=path{
+       refactor.rb
+      }
       util.rb
       with_key.rb
       rake.rb

data/VERSION

@@ -1 +1 @@
-7.1.0
+7.3.0

data/bin/scout

@@ -37,6 +37,11 @@ if dev_dir
       $LOAD_PATH.unshift f
     end
   end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
 end
 
 Log.nocolor = true if ARGV.include? "--nocolor"
@@ -112,7 +117,6 @@ def commands(prev)
 end
 
 def scout_usage(prev = nil)
-  puts
   puts SOPT.doc
 
   if prev

data/lib/rbbt-scout.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
+module Rbbt
+  extend Resource
+  self.pkgdir = 'rbbt'
+end

data/lib/scout/concurrent_stream.rb

@@ -9,11 +9,11 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
     
-    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
+    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -24,6 +24,7 @@ module ConcurrentStream
     stream.no_fail = no_fail unless no_fail.nil?
     stream.std_err = ""
 
+    stream.next = next_stream unless next_stream.nil?
     stream.pair = pair unless pair.nil?
 
     callback = block if block_given?
@@ -53,7 +54,7 @@ module ConcurrentStream
 
     stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
 
-    stream.lockfile = lockfile unless lockfile.nil?
+    stream.lock = lock unless lock.nil?
 
     stream.aborted = false
 
@@ -61,7 +62,7 @@ module ConcurrentStream
   end
 
   def annotate(stream)
-    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
     stream
   end
 
@@ -135,11 +136,12 @@ module ConcurrentStream
     begin
       join_threads
       join_pids
+      raise stream_exception if stream_exception
       join_callback
       close unless closed?
     ensure
       @joined = true
-      lockfile.unlock if lockfile and lockfile.locked?
+      lock.unlock if lock and lock.locked?
       raise stream_exception if stream_exception
     end
   end
@@ -159,7 +161,7 @@ module ConcurrentStream
     @threads.each do |t|
       next if t == Thread.current
       begin
-        t.join unless t == Thread.current
+        t.join
       rescue Aborted
       rescue Exception
         Log.debug "Thread (#{name}) exception: #{$!.message}"
@@ -204,8 +206,8 @@ module ConcurrentStream
     ensure
       close unless closed?
 
-      if lockfile and lockfile.locked?
-        lockfile.unlock 
+      if lock and lock.locked?
+        lock.unlock 
       end
     end
   end
@@ -230,6 +232,9 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
+    rescue
+      raise stream_exception if stream_exception
+      raise $!
     ensure
       begin
         close unless closed?

data/lib/scout/config.rb

@@ -0,0 +1,168 @@
+require_relative 'path'
+require_relative 'resource'
+require_relative 'resource/scout'
+
+module Scout::Config
+
+  CACHE = IndiferentHash.setup({})
+
+  GOT_KEYS=[]
+
+  def self.add_entry(key, value, tokens)
+    tokens = [tokens] unless Array === tokens
+    tokens << "key:#{key}" unless tokens.include?("key:#{key}")
+    CACHE[key.to_s] ||= [] 
+    CACHE[key.to_s] << [tokens, value]
+  end
+
+  def self.load_file(file)
+    Log.debug "Loading config file: #{ file }"
+    TSV.traverse file, :type => :array do |line|
+      next if line =~ /^#/
+      key, value, *tokens = line.strip.split(/\s/)
+
+      self.add_entry(key, value, tokens) if key
+    end
+  end
+
+  def self.load_config
+    Scout.etc.config.find_all.reverse.each do |file|
+      self.load_file(file)
+    end
+  end
+
+  def self.set(values, *tokens)
+    if not Hash === values
+      values = {values => tokens.shift}
+    end
+
+    values.each do |key,value|
+      add_entry key, value, tokens
+    end
+  end
+
+  def self.token_priority(token)
+    token, _sep, priority = token.to_s.partition("::")
+
+    if priority.nil? || priority.empty?
+      type, _sep, rest = token.partition(":")
+      priority = case type
+                 when "workflow"
+                   4
+                 when "task"
+                   3
+                 when "file"
+                   2
+                 when "line"
+                   1
+                 when "key"
+                   20
+                 else
+                   10
+                 end
+    else
+      priority = priority.to_i
+    end
+
+    [token, priority]
+  end
+
+  def self.match(entries, give_token)
+    priorities = {}
+    entries.each do |tokens, value|
+      best_prio = nil
+      tokens = [tokens] unless Array === tokens
+      tokens.each do |tok|
+        tok, prio = token_priority tok
+        next unless tok == give_token
+
+        best_prio = prio if best_prio.nil? or best_prio > prio
+        next if prio > best_prio
+
+        priorities[prio] ||= []
+        priorities[prio].unshift value
+      end
+    end if entries
+    priorities
+  end
+
+  # For equal priorities the matching prioritizes tokens ealier in the list
+  def self.get(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    default = options.nil? ? nil : options[:default]
+
+    tokens = ["key:" + key] if tokens.empty?
+
+    tokens = tokens.flatten
+    file, _sep, line = caller.reject{|l| 
+      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+        l =~ /rbbt\/resource\/path\.rb/ or
+        l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /accessor\.rb/ or
+        l =~ /progress-monitor\.rb/ 
+    }.first.partition(":")
+
+    File.expand_path(file)
+
+    tokens << ("file:" << file)
+    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
+
+    entries = CACHE[key.to_s]
+    priorities = {}
+    tokens.each do |token|
+      token_prio = match entries, token.to_s
+      token_prio.each do |prio, values|
+        priorities[prio] ||= []
+        priorities[prio].concat(values)
+      end
+    end
+
+    value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
+    value = false if value == 'false'
+
+    Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
+    GOT_KEYS << [key, value, tokens]
+
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    elsif value == 'nil'
+      nil
+    else
+      value
+    end
+  end
+
+  def self.with_config
+      saved_config = {}
+      CACHE.each do |k,v|
+        saved_config[k] = v.dup
+      end
+      saved_got_keys = GOT_KEYS.dup
+    begin
+      yield
+    ensure
+      CACHE.replace(saved_config)
+      GOT_KEYS.replace(saved_got_keys)
+    end
+  end
+
+  def self.process_config(config)
+    if Misc.is_filename?(config) && File.exist?(config)
+      Scout::Config.load_file(config)
+    elsif Scout.etc.config_profile[config].exists?
+      Scout::Config.load_file(Scout.etc.config_profile[config].find)
+    else
+      key, value, *tokens = config.split(/\s/)
+      tokens = tokens.collect do |tok|
+        tok, _sep, prio = tok.partition("::")
+        prio = "0" if prio.nil? or prio.empty?
+        [tok, prio] * "::"
+      end
+      Scout::Config.set({key => value}, *tokens)
+    end
+  end
+
+
+  self.load_config
+end

data/lib/scout/exceptions.rb

@@ -85,13 +85,14 @@ class DoneProcessing < Exception
 end
 
 class WorkerException < ScoutException
-  attr_accessor :exception, :pid
-  def initialize(exception, pid)
-    @exception = exception
+  attr_accessor :worker_exception, :pid
+  def initialize(worker_exception, pid)
+    @worker_exception = worker_exception
     @pid = pid
   end
 end
 
+class SemaphoreInterrupted < TryAgain; end
 
 #class OpenGzipError < StandardError; end
 #
@@ -137,3 +138,4 @@ end
 #
 #
 #
+class ResourceNotFound < ScoutException; end

data/lib/scout/indiferent_hash/options.rb

@@ -29,6 +29,7 @@ module IndiferentHash
 
   def self.pull_keys(hash, prefix)
     new = {}
+    prefix = prefix.to_s
     hash.keys.each do |key|
       if key.to_s =~ /#{ prefix }_(.*)/
         case

data/lib/scout/indiferent_hash.rb

@@ -58,9 +58,11 @@ module IndiferentHash
   def delete(key)
     case key
     when Symbol, Module
-      super(key) || super(key.to_s)
+      v = super(key) 
+      v.nil? ? super(key.to_s) : v
     when String
-      super(key) || super(key.to_sym)
+      v = super(key)
+      v.nil? ? super(key.to_sym) : v
     else
       super(key)
     end

data/lib/scout/log/color.rb

@@ -2,7 +2,6 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
-require 'colorist'
 
 module Colorize
   def self.colors=(colors)
@@ -185,8 +184,10 @@ module Log
                 :red
               when :waiting, :queued
                 :yellow
-              when :started, :start, :streamming
+              when :started, :streamming
                 :cyan
+              when :start
+                :title
               else
                 :cyan
               end

data/lib/scout/log/progress/report.rb

@@ -2,6 +2,7 @@ require 'yaml'
 module Log
   class ProgressBar
     def print(io, str)
+      return if self.severity && self.severity < Log.severity
       return if Log.no_bar
       STDERR.print str
       Log.logfile.puts str unless Log.logfile.nil?

data/lib/scout/log/progress/util.rb

@@ -81,7 +81,7 @@ module Log
       Log::ProgressBar.remove_bar self, error
     end
 
-    def self.with_bar(max, options = {})
+    def self.with_bar(max = nil, options = {})
       bar = new_bar(max, options)
       begin
         error = false
@@ -96,6 +96,71 @@ module Log
         remove_bar(bar, error) if bar && ! keep
       end
     end
+
+    def self.guess_obj_max(obj)
+    begin
+      case obj
+      when (defined? Step and Step)
+        if obj.done?
+          path = obj.path
+          path = path.find if path.respond_to? :find
+          if File.exist? path
+            CMD.cmd("wc -l '#{path}'").read.to_i 
+          else
+            nil
+          end
+        else
+          nil
+        end
+      when TSV
+        obj.length
+      when Array, Hash
+        obj.size
+      when File
+        return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+        CMD.cmd("wc -l '#{obj.path}'").read.to_i
+      when Path, String
+        obj = obj.find if Path === obj
+        if File.exist? obj
+          return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+          CMD.cmd("wc -l '#{obj}'").read.to_i
+        else
+          nil
+        end
+      end
+    rescue Exception
+      Log.exception $!
+      nil
+    end
+  end
+
+  def self.get_obj_bar(bar, obj)
+    case bar
+    when String
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(max, {:desc => bar}) 
+    when TrueClass
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(max, nil) 
+    when Numeric
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(bar) 
+    when Hash
+      max = Misc.process_options(bar, :max) || max
+      Log::ProgressBar.new_bar(max, bar) 
+    when Log::ProgressBar
+      bar.max ||= guess_obj_max(obj)
+      bar
+    else
+      if (defined? Step and Step === bar)
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+      else
+        bar
+      end
+    end
+  end
   end
+
 end
 

data/lib/scout/log/progress.rb

@@ -15,20 +15,22 @@ module Log
 
     class << self
       attr_accessor :default_file
+      attr_accessor :default_severity
     end
 
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
 
     def initialize(max = nil, options = {})
-      depth, desc, file, bytes, frequency, process, callback = 
-        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
-        :depth => 0, :frequency => 2
+      depth, desc, file, bytes, frequency, process, callback, severity = 
+        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback, :severity,
+        :depth => 0, :frequency => 2, :severity => Log::ProgressBar.default_severity
 
       max = nil if TrueClass === max
 
       @max = max
       @ticks = 0
       @frequency = frequency
+      @severity = severity
       @last_time = nil
       @last_count = nil
       @last_percent = nil

data/lib/scout/log.rb

@@ -186,11 +186,12 @@ module Log
 
   def self.exception(e)
     stack = caller
+    backtrace = e.backtrace || []
     if ENV["RBBT_ORIGINAL_STACK"] == 'true'
       error([e.class.to_s, e.message].compact * ": " )
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
     else
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace.reverse)*"\n")
       error([e.class.to_s, e.message].compact * ": " )
     end
   end

data/lib/scout/misc/helper.rb

@@ -0,0 +1,31 @@
+module Misc
+  def self.intersect_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    intersect = []
+    while true
+      break if e1.nil? or e2.nil?
+      case e1 <=> e2
+      when 0
+        intersect << e1
+        e1, e2 = a1.shift, a2.shift
+      when -1
+        e1 = a1.shift while not e1.nil? and e1 < e2
+      when 1
+        e2 = a2.shift
+        e2 = a2.shift while not e2.nil? and e2 < e1
+      end
+    end
+    intersect
+  end
+
+  def self.counts(array)
+    counts = {}
+    array.each do |e|
+      counts[e] ||= 0
+      counts[e] += 1
+    end
+
+    counts
+  end
+
+end

data/lib/scout/misc/monitor.rb

@@ -1,4 +1,7 @@
 module Misc
+  def self.pid_alive?(pid)
+    !! Process.kill(0, pid) rescue false
+  end
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil
@@ -11,7 +14,7 @@ module Misc
       if message
         puts "#{message }: #{ repeats } repeats"
       else
-        puts "Benchmark for #{ repeats } repeats"
+        puts "Benchmark for #{ repeats } repeats (#{caller.first})"
       end
       puts measure
     rescue Exception

data/lib/scout/misc/system.rb

@@ -0,0 +1,15 @@
+module Misc
+  def self.env_add(var, value, sep = ":", prepend = true)
+    if ENV[var].nil?
+      ENV[var] = value
+    elsif ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
+      return
+    else
+      if prepend
+        ENV[var] = value + sep + ENV[var]
+      else
+        ENV[var] += sep + value
+      end
+    end
+  end
+end

data/lib/scout/misc.rb

@@ -3,6 +3,8 @@ require_relative 'misc/insist'
 require_relative 'misc/digest'
 require_relative 'misc/filesystem'
 require_relative 'misc/monitor'
+require_relative 'misc/system'
+require_relative 'misc/helper'
 
 module Misc
 end