scout-gear 7.1.0 → 7.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d00b2f7d1a5156c6259729b245722979b34132ca3a66922110662fadfc56b65
-  data.tar.gz: 057a511570907a68d6ee7d962705bfe3cc92325c1fc026b45c5ff0dfb0634f96
+  metadata.gz: de499ccbdffd888298324a025b253f0dd0e359d279d6abe8331acd91fd00cc57
+  data.tar.gz: 9643053080da23a98209ebefbcd3b3043b7031374a405ffed112460f1ddcf411
 SHA512:
-  metadata.gz: 8ca8280c5f4ac63ca98cc8c2388700e29752ec37b008d3f9b54a7bab13569af24fc94afbbfc60d9e295ca81c9a2e5a64bdab83e6cab7c9dd6d531013d72f6954
-  data.tar.gz: c056280d255116ed9a3c74ee8c1b4c367f30abf6ec60d278180534afc1728d104f89ac247fa0b1d7fb751692b869e2cf73a6a8ac8b6bef8f07e411a59e476409
+  metadata.gz: 73afbf6aeb4032380fc45e2f60e2e9335d763c6837ef3bb8247c83bfaf2cd962a6fdfba767788b988e00f685d0e9bc19927a4e794fc092296cc088f23d4565ee
+  data.tar.gz: 309e86afe466c649800fd212fd22e9e183a5e92ba558d88ec0258690a15437952ead3e968c3693b89c036dc2fa9e27f4d129f6be6c001031dc38ab071361ba99

data/.vimproject

@@ -18,6 +18,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  lib=lib {
   scout-gear.rb
   workflow-scout.rb
+  rbbt-scout.rb
   scout.rb
   scout=scout{
    meta_extension.rb
@@ -30,7 +31,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     digest.rb
     filesystem.rb
     monitor.rb
+    system.rb
    }
+   named_array.rb
    indiferent_hash.rb
    indiferent_hash=indiferent_hash{
     case_insensitive.rb
@@ -77,10 +80,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     produce=produce{
      rake.rb
     }
+    software.rb
     scout.rb
     util.rb
     path.rb
    }
+   config.rb
    persist.rb
    persist=persist{
     serialize.rb
@@ -94,6 +99,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     step=step{
      info.rb
      load.rb
+     file.rb
+     config.rb
+     dependencies.rb
+     provenance.rb
     }
     task.rb
     task=task{
@@ -112,6 +121,16 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    tsv.rb
    tsv=tsv{
     parser.rb
+    dumper.rb
+    persist.rb
+    index.rb
+    persist=persist{
+     adapter.rb
+     tokyocabinet.rb
+    }
+    traverse.rb
+    util.rb
+    path.rb
    }
   }
  }
@@ -126,6 +145,13 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    test_tmpfile.rb
   }
  }
+ share=share{
+  software=software{
+   install_helpers
+  }
+ }
+
+
 
  modules=modules{
   rbbt-util=rbbt-util filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.txt *.conf LICENSE" {
@@ -386,6 +412,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      resource.rb
      resource=resource{
       path.rb
+      path=path{
+       refactor.rb
+      }
       util.rb
       with_key.rb
       rake.rb

data/VERSION

@@ -1 +1 @@
-7.1.0
+7.2.0

data/bin/scout

@@ -37,6 +37,11 @@ if dev_dir
       $LOAD_PATH.unshift f
     end
   end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
 end
 
 Log.nocolor = true if ARGV.include? "--nocolor"
@@ -112,7 +117,6 @@ def commands(prev)
 end
 
 def scout_usage(prev = nil)
-  puts
   puts SOPT.doc
 
   if prev

data/lib/rbbt-scout.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
+module Rbbt
+  extend Resource
+  self.pkgdir = 'rbbt'
+end

data/lib/scout/concurrent_stream.rb

@@ -9,11 +9,11 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
     
-    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
+    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -24,6 +24,7 @@ module ConcurrentStream
     stream.no_fail = no_fail unless no_fail.nil?
     stream.std_err = ""
 
+    stream.next = next_stream unless next_stream.nil?
     stream.pair = pair unless pair.nil?
 
     callback = block if block_given?
@@ -230,6 +231,9 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
+    rescue
+      raise stream_exception if stream_exception
+      raise $!
     ensure
       begin
         close unless closed?

data/lib/scout/config.rb

@@ -0,0 +1,168 @@
+require_relative 'path'
+require_relative 'resource'
+require_relative 'resource/scout'
+
+module Scout::Config
+
+  CACHE = IndiferentHash.setup({})
+
+  GOT_KEYS=[]
+
+  def self.add_entry(key, value, tokens)
+    tokens = [tokens] unless Array === tokens
+    tokens << "key:#{key}" unless tokens.include?("key:#{key}")
+    CACHE[key.to_s] ||= [] 
+    CACHE[key.to_s] << [tokens, value]
+  end
+
+  def self.load_file(file)
+    Log.debug "Loading config file: #{ file }"
+    TSV.traverse file, :type => :array do |line|
+      next if line =~ /^#/
+      key, value, *tokens = line.strip.split(/\s/)
+
+      self.add_entry(key, value, tokens) if key
+    end
+  end
+
+  def self.load_config
+    Scout.etc.config.find_all.reverse.each do |file|
+      self.load_file(file)
+    end
+  end
+
+  def self.set(values, *tokens)
+    if not Hash === values
+      values = {values => tokens.shift}
+    end
+
+    values.each do |key,value|
+      add_entry key, value, tokens
+    end
+  end
+
+  def self.token_priority(token)
+    token, _sep, priority = token.to_s.partition("::")
+
+    if priority.nil? || priority.empty?
+      type, _sep, rest = token.partition(":")
+      priority = case type
+                 when "workflow"
+                   4
+                 when "task"
+                   3
+                 when "file"
+                   2
+                 when "line"
+                   1
+                 when "key"
+                   20
+                 else
+                   10
+                 end
+    else
+      priority = priority.to_i
+    end
+
+    [token, priority]
+  end
+
+  def self.match(entries, give_token)
+    priorities = {}
+    entries.each do |tokens, value|
+      best_prio = nil
+      tokens = [tokens] unless Array === tokens
+      tokens.each do |tok|
+        tok, prio = token_priority tok
+        next unless tok == give_token
+
+        best_prio = prio if best_prio.nil? or best_prio > prio
+        next if prio > best_prio
+
+        priorities[prio] ||= []
+        priorities[prio].unshift value
+      end
+    end if entries
+    priorities
+  end
+
+  # For equal priorities the matching prioritizes tokens ealier in the list
+  def self.get(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    default = options.nil? ? nil : options[:default]
+
+    tokens = ["key:" + key] if tokens.empty?
+
+    tokens = tokens.flatten
+    file, _sep, line = caller.reject{|l| 
+      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+        l =~ /rbbt\/resource\/path\.rb/ or
+        l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /accessor\.rb/ or
+        l =~ /progress-monitor\.rb/ 
+    }.first.partition(":")
+
+    File.expand_path(file)
+
+    tokens << ("file:" << file)
+    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
+
+    entries = CACHE[key.to_s]
+    priorities = {}
+    tokens.each do |token|
+      token_prio = match entries, token.to_s
+      token_prio.each do |prio, values|
+        priorities[prio] ||= []
+        priorities[prio].concat(values)
+      end
+    end
+
+    value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
+    value = false if value == 'false'
+
+    Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
+    GOT_KEYS << [key, value, tokens]
+
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    elsif value == 'nil'
+      nil
+    else
+      value
+    end
+  end
+
+  def self.with_config
+      saved_config = {}
+      CACHE.each do |k,v|
+        saved_config[k] = v.dup
+      end
+      saved_got_keys = GOT_KEYS.dup
+    begin
+      yield
+    ensure
+      CACHE.replace(saved_config)
+      GOT_KEYS.replace(saved_got_keys)
+    end
+  end
+
+  def self.process_config(config)
+    if Misc.is_filename?(config) && File.exist?(config)
+      Scout::Config.load_file(config)
+    elsif Scout.etc.config_profile[config].exists?
+      Scout::Config.load_file(Scout.etc.config_profile[config].find)
+    else
+      key, value, *tokens = config.split(/\s/)
+      tokens = tokens.collect do |tok|
+        tok, _sep, prio = tok.partition("::")
+        prio = "0" if prio.nil? or prio.empty?
+        [tok, prio] * "::"
+      end
+      Scout::Config.set({key => value}, *tokens)
+    end
+  end
+
+
+  self.load_config
+end

data/lib/scout/exceptions.rb

@@ -85,9 +85,9 @@ class DoneProcessing < Exception
 end
 
 class WorkerException < ScoutException
-  attr_accessor :exception, :pid
-  def initialize(exception, pid)
-    @exception = exception
+  attr_accessor :worker_exception, :pid
+  def initialize(worker_exception, pid)
+    @worker_exception = worker_exception
     @pid = pid
   end
 end
@@ -137,3 +137,4 @@ end
 #
 #
 #
+class ResourceNotFound < ScoutException; end

data/lib/scout/indiferent_hash/options.rb

@@ -29,6 +29,7 @@ module IndiferentHash
 
   def self.pull_keys(hash, prefix)
     new = {}
+    prefix = prefix.to_s
     hash.keys.each do |key|
       if key.to_s =~ /#{ prefix }_(.*)/
         case

data/lib/scout/indiferent_hash.rb

@@ -58,9 +58,11 @@ module IndiferentHash
   def delete(key)
     case key
     when Symbol, Module
-      super(key) || super(key.to_s)
+      v = super(key) 
+      v.nil? ? super(key.to_s) : v
     when String
-      super(key) || super(key.to_sym)
+      v = super(key)
+      v.nil? ? super(key.to_sym) : v
     else
       super(key)
     end

data/lib/scout/log/color.rb

@@ -185,8 +185,10 @@ module Log
                 :red
               when :waiting, :queued
                 :yellow
-              when :started, :start, :streamming
+              when :started, :streamming
                 :cyan
+              when :start
+                :title
               else
                 :cyan
               end

data/lib/scout/log/progress/report.rb

@@ -2,6 +2,7 @@ require 'yaml'
 module Log
   class ProgressBar
     def print(io, str)
+      return if self.severity && self.severity < Log.severity
       return if Log.no_bar
       STDERR.print str
       Log.logfile.puts str unless Log.logfile.nil?

data/lib/scout/log/progress/util.rb

@@ -81,7 +81,7 @@ module Log
       Log::ProgressBar.remove_bar self, error
     end
 
-    def self.with_bar(max, options = {})
+    def self.with_bar(max = nil, options = {})
       bar = new_bar(max, options)
       begin
         error = false

data/lib/scout/log/progress.rb

@@ -15,20 +15,22 @@ module Log
 
     class << self
       attr_accessor :default_file
+      attr_accessor :default_severity
     end
 
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
 
     def initialize(max = nil, options = {})
-      depth, desc, file, bytes, frequency, process, callback = 
-        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
-        :depth => 0, :frequency => 2
+      depth, desc, file, bytes, frequency, process, callback, severity = 
+        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback, :severity,
+        :depth => 0, :frequency => 2, :severity => Log::ProgressBar.default_severity
 
       max = nil if TrueClass === max
 
       @max = max
       @ticks = 0
       @frequency = frequency
+      @severity = severity
       @last_time = nil
       @last_count = nil
       @last_percent = nil

data/lib/scout/log.rb

@@ -186,11 +186,12 @@ module Log
 
   def self.exception(e)
     stack = caller
+    backtrace = e.backtrace || []
     if ENV["RBBT_ORIGINAL_STACK"] == 'true'
       error([e.class.to_s, e.message].compact * ": " )
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
     else
-      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
+      error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace.reverse)*"\n")
       error([e.class.to_s, e.message].compact * ": " )
     end
   end

data/lib/scout/misc/monitor.rb

@@ -1,4 +1,7 @@
 module Misc
+  def self.pid_alive?(pid)
+    !! Process.kill(0, pid) rescue false
+  end
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil

data/lib/scout/misc/system.rb

@@ -0,0 +1,15 @@
+module Misc
+  def self.env_add(var, value, sep = ":", prepend = true)
+    if ENV[var].nil?
+      ENV[var] = value
+    elsif ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
+      return
+    else
+      if prepend
+        ENV[var] = value + sep + ENV[var]
+      else
+        ENV[var] += sep + value
+      end
+    end
+  end
+end

data/lib/scout/misc.rb

@@ -3,6 +3,7 @@ require_relative 'misc/insist'
 require_relative 'misc/digest'
 require_relative 'misc/filesystem'
 require_relative 'misc/monitor'
+require_relative 'misc/system'
 
 module Misc
 end

data/lib/scout/named_array.rb

@@ -0,0 +1,68 @@
+require_relative 'meta_extension'
+module NamedArray
+  extend MetaExtension
+  extension_attr :fields
+
+  def self.identify_name(names, selected)
+    res = (Array === selected ? selected : [selected]).collect do |field|
+      case field
+      when nil
+        0
+      when Integer
+        field
+      when Symbol
+        field == :key ? field : identify_name(names, field.to_s)
+      when (names.nil? and String)
+        if field =~ /^\d+$/
+          identify_field(key_field, fields, field.to_i)
+        else
+          raise "No name information available and specified name not numeric: #{ field }"
+        end
+      when Symbol
+        names.index{|f| f.to_s == field.to_s }
+      when String
+        pos = names.index{|f| f.to_s == field }
+        next pos if pos
+        if field =~ /^\d+$/
+          next identify_names(names, field.to_i)
+        end
+        pos = names.index{|name| name.start_with?(field) }
+        next pos if pos
+        nil
+      else
+        raise "Field '#{ Log.fingerprint field }' was not understood. Options: (#{ Log.fingerprint names })"
+      end
+    end
+
+    Array === selected ? res : res.first
+  end
+
+  def positions(fields)
+    if Array ==  fields
+      fields.collect{|field|
+        NamedArray.identify_name(@fields, field)
+      }
+    else
+      NamedArray.identify_name(@fields, fields)
+    end
+  end
+
+  def [](key)
+    pos = NamedArray.identify_name(@fields, key)
+    super(pos)
+  end
+
+  def concat(other)
+    super(other)
+    self.fields.concat(other.fields) if NamedArray === other
+    self
+  end
+
+  def to_hash
+    hash = {}
+    self.fields.zip(self) do |field,value|
+      hash[field] = value
+    end
+    IndiferentHash.setup hash
+  end
+end

data/lib/scout/open/stream.rb

@@ -1,5 +1,5 @@
 module Open
-  BLOCK_SIZE = 1024
+  BLOCK_SIZE = 1024 * 8
 
   class << self
     attr_accessor :sensible_write_lock_dir
@@ -52,9 +52,7 @@ module Open
           into_path, into = into, File.open(into, 'w') 
         end
         
-        into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
-        io.sync = true
 
         begin
           while c = io.readpartial(BLOCK_SIZE)
@@ -77,7 +75,7 @@ module Open
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
         Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
-        exception = io.stream_exception || $!
+        exception = (io.respond_to?(:stream_exception) && io.stream_exception) ? io.stream_exception : $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         into_path = into if into_path.nil? && String === into
@@ -110,7 +108,7 @@ module Open
         Log.warn "Path exists in sensible_write, not forcing update: #{ path }"
         Open.consume_stream content 
       else
-        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
+        FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory?(File.dirname(tmp_path))
         FileUtils.rm_f tmp_path if File.exist? tmp_path
         begin
 
@@ -121,7 +119,7 @@ module Open
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
-              f.sync = true
+              #f.sync = true
               begin
                 while block = content.readpartial(BLOCK_SIZE)
                   f.write block
@@ -345,7 +343,6 @@ module Open
       end
     end
 
-
     out_pipes.each do |sout|
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
     end
@@ -400,5 +397,39 @@ module Open
     end
     str
   end
+  
+  def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
+    Open.open_pipe do |sin|
+      line = stream.gets
+      while line =~ /^#{header_hash}/ do
+        sin.puts line
+        line = stream.gets
+      end
+
+      line_stream = Open.open_pipe do |line_stream_in|
+        line_stream_in.puts line
+        begin
+          Open.consume_stream(stream, false, line_stream_in)
+        rescue
+          raise $!
+        end
+      end
+
+      sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+
+      begin
+        Open.consume_stream(sorted, false, sin)
+      rescue
+        Log.exception $!
+        begin
+          sorted.raise($!) if sorted.respond_to? :raise
+          stream.raise($!) if stream.respond_to? :raise
+        ensure
+          raise $!
+        end
+      end
+    end
+  end
+
 
 end

data/lib/scout/path/find.rb

@@ -9,7 +9,8 @@ module Path
         break unless file =~ /(?:scout|rbbt)\/(?:resource\.rb|workflow\.rb)/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\.rb/ or
           file =~ /(?:scout|rbbt)\/(?:.*\/)?path\/(?:find|refactor|util)\.rb/ or
-          file =~ /(?:scout|rbbt)\/persist.rb/
+          file =~ /(?:scout|rbbt)\/persist.rb/ or
+          file =~ /modules\/rbbt-util/
       end
       file = file.sub(/\.rb[^\w].*/,'.rb')
     end
@@ -136,15 +137,38 @@ module Path
     found
   end
 
+  def self.exists_file_or_alternatives(file)
+    return file if File.exist?(file) or File.directory?(file)
+    %w(gz bgz zip).each do |extension|
+      alt_file = file + '.' + extension
+      return alt_file if File.exist?(alt_file) or File.directory?(alt_file)
+    end
+    nil
+  end
+
   def find(where = nil)
-    return self if located?
+    if located?
+      if File.exist?(self)
+        return self if located?
+      else
+        found = Path.exists_file_or_alternatives(self)
+        if found
+          return self.annotate(found)
+        else
+          return self if located?
+        end
+      end
+    end
+
     return find_all if where == 'all' || where == :all
+
     return follow(where) if where
 
     map_order.each do |map_name|
       found = follow(map_name, false)
 
-      return annotate_found_where(found, map_name) if File.exist?(found) || File.directory?(found)
+      found = Path.exists_file_or_alternatives(found)
+      return annotate_found_where(found, map_name) if found
     end
 
     return follow(:default)