scout-essentials 1.0.0

data/lib/scout/concurrent_stream.rb

@@ -0,0 +1,284 @@
+require_relative 'indiferent_hash'
+
+module AbortedStream
+  attr_accessor :exception
+  def self.setup(obj, exception = nil)
+    obj.extend AbortedStream
+    obj.exception = exception
+  end
+end
+
+module ConcurrentStream
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
+
+  def self.setup(stream, options = {}, &block)
+    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+    stream.autojoin = autojoin unless autojoin.nil?
+    stream.no_fail = no_fail unless no_fail.nil?
+    stream.std_err = ""
+
+    stream.next = next_stream unless next_stream.nil?
+    stream.pair = pair unless pair.nil?
+
+    callback = block if block_given?
+    if callback
+      if stream.callback
+        old_callback = stream.callback
+        stream.callback = Proc.new do
+          old_callback.call
+          callback.call
+        end
+      else
+        stream.callback = callback
+      end
+    end
+
+    if abort_callback
+      if stream.abort_callback
+        old_abort_callback = stream.abort_callback
+        stream.abort_callback = Proc.new do
+          old_abort_callback.call
+          abort_callback.call
+        end
+      else
+        stream.abort_callback = abort_callback
+      end
+    end
+
+    stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
+
+    stream.lock = lock unless lock.nil?
+
+    stream.aborted = false
+
+    stream
+  end
+
+  def annotate(stream)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
+    stream
+  end
+
+  def clear
+    @threads = @pids = @callback = @abort_callback = @joined = nil
+  end
+
+  def joined?
+    @joined
+  end
+
+  def aborted?
+    @aborted
+  end
+
+  def join_threads
+    if @threads
+      @threads.each do |t|
+        next if t == Thread.current
+        begin
+          t.join
+          if Process::Status === t.value
+            if ! (t.value.success? || no_fail)
+
+              if log
+                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+              else
+                msg = "Error joining #{self.filename || self.inspect}"
+              end
+
+              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
+            end
+          end
+        rescue Exception
+          if no_fail
+            Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
+          else
+            Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
+            stream_raise_exception $!
+          end
+        end
+      end
+    end
+    @threads = []
+  end
+
+  def join_pids
+    if @pids and @pids.any?
+      @pids.each do |pid|
+        begin
+          Process.waitpid(pid, Process::WUNTRACED)
+          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
+        rescue Errno::ECHILD
+        end
+      end
+      @pids = []
+    end
+  end
+
+  def join_callback
+    if @callback and not joined?
+      begin
+        @callback.call
+      ensure
+        @callback = nil
+      end
+    end
+  end
+
+  def join
+    begin
+      join_threads
+      join_pids
+      raise stream_exception if stream_exception
+      join_callback
+      close unless closed?
+    ensure
+      @joined = true
+      begin
+        lock.unlock if lock && lock.locked?
+      rescue
+        Log.exception $!
+      end
+      raise stream_exception if stream_exception
+    end
+  end
+
+  def abort_threads(exception = nil)
+    return unless @threads and @threads.any?
+    name = Log.fingerprint(Thread.current)
+    name += " - file:#{filename}" if filename
+    Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
+
+    threads = @threads.dup
+    @threads.clear
+    threads.each do |t|
+      next if t == Thread.current
+      next if t["aborted"]
+      t["aborted"] = true
+      exception = exception.nil? ? Aborted.new : exception
+      Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
+      t.raise(exception)
+      t.join
+    end
+  end
+
+  def abort_pids
+    @pids.each do |pid|
+      begin
+        Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
+        Process.kill :INT, pid
+      rescue Errno::ESRCH
+      end
+    end if @pids
+    @pids = []
+  end
+
+  def abort(exception = nil)
+    self.stream_exception ||= exception
+    if @aborted
+      Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
+      return
+    else
+      Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
+    end
+    AbortedStream.setup(self, exception)
+    @aborted = true
+    begin
+      @abort_callback.call exception if @abort_callback
+
+      abort_threads(exception)
+      abort_pids
+
+      @callback = nil
+      @abort_callback = nil
+
+      if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
+        Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
+        @pair.abort exception
+      end
+    ensure
+      close unless closed?
+
+      if lock and lock.locked?
+        lock.unlock
+      end
+    end
+  end
+
+  def close(*args)
+    if autojoin
+      begin
+        super(*args)
+      rescue
+        self.abort
+        self.join
+        stream_raise_exception $!
+      ensure
+        self.join if ! @stream_exception && (self.closed? || self.eof?)
+      end
+    else
+      super(*args)
+    end
+  end
+
+  def read(*args)
+    begin
+      super(*args)
+    rescue Exception
+      @stream_exception ||= $!
+      raise @stream_exception
+    ensure
+      if ! @stream_exception && autojoin && ! closed?
+        begin
+          done = eof?
+        rescue Exception
+          self.abort($!)
+          raise $!
+        end
+        close if done
+      end
+    end
+  end
+
+  def add_callback(&block)
+    old_callback = callback
+    @callback = Proc.new do
+      old_callback.call if old_callback
+      block.call
+    end
+  end
+
+  def stream_raise_exception(exception)
+    self.stream_exception = exception
+    threads.each do |thread|
+      thread.raise exception
+    end
+    self.abort
+  end
+
+  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
+    ConcurrentStream.setup(stream, **kwargs)
+    begin
+      begin
+        yield
+      ensure
+        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
+        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
+      end
+    rescue Aborted
+      Log.low "Aborted #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    rescue Exception
+      Log.low "Exception #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    end
+  end
+end

data/lib/scout/config.rb

@@ -0,0 +1,168 @@
+require_relative 'path'
+require_relative 'resource'
+require_relative 'resource/scout'
+
+module Scout::Config
+
+  CACHE ||= IndiferentHash.setup({})
+
+  GOT_KEYS=[]
+
+  def self.add_entry(key, value, tokens)
+    tokens = [tokens] unless Array === tokens
+    tokens << "key:#{key}" unless tokens.include?("key:#{key}")
+    CACHE[key.to_s] ||= [] 
+    CACHE[key.to_s] << [tokens, value]
+  end
+
+  def self.load_file(file)
+    Log.debug "Loading config file: #{ file }"
+    TSV.traverse file, :type => :array do |line|
+      next if line =~ /^#/
+      key, value, *tokens = line.strip.split(/\s/)
+
+      self.add_entry(key, value, tokens) if key
+    end
+  end
+
+  def self.load_config
+    Scout.etc.config.find_all.reverse.each do |file|
+      self.load_file(file)
+    end
+  end
+
+  def self.set(values, *tokens)
+    if not Hash === values
+      values = {values => tokens.shift}
+    end
+
+    values.each do |key,value|
+      add_entry key, value, tokens
+    end
+  end
+
+  def self.token_priority(token)
+    token, _sep, priority = token.to_s.partition("::")
+
+    if priority.nil? || priority.empty?
+      type, _sep, rest = token.partition(":")
+      priority = case type
+                 when "workflow"
+                   4
+                 when "task"
+                   3
+                 when "file"
+                   2
+                 when "line"
+                   1
+                 when "key"
+                   20
+                 else
+                   10
+                 end
+    else
+      priority = priority.to_i
+    end
+
+    [token, priority]
+  end
+
+  def self.match(entries, give_token)
+    priorities = {}
+    entries.each do |tokens, value|
+      best_prio = nil
+      tokens = [tokens] unless Array === tokens
+      tokens.each do |tok|
+        tok, prio = token_priority tok
+        next unless tok == give_token
+
+        best_prio = prio if best_prio.nil? or best_prio > prio
+        next if prio > best_prio
+
+        priorities[prio] ||= []
+        priorities[prio].unshift value
+      end
+    end if entries
+    priorities
+  end
+
+  # For equal priorities the matching prioritizes tokens ealier in the list
+  def self.get(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    default = options.nil? ? nil : options[:default]
+
+    tokens = ["key:" + key] if tokens.empty?
+
+    tokens = tokens.flatten
+    file, _sep, line = caller.reject{|l| 
+      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
+        l =~ /rbbt\/resource\/path\.rb/ or
+        l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /accessor\.rb/ or
+        l =~ /progress-monitor\.rb/ 
+    }.first.partition(":")
+
+    File.expand_path(file)
+
+    tokens << ("file:" << file)
+    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
+
+    entries = CACHE[key.to_s]
+    priorities = {}
+    tokens.each do |token|
+      token_prio = match entries, token.to_s
+      token_prio.each do |prio, values|
+        priorities[prio] ||= []
+        priorities[prio].concat(values)
+      end
+    end
+
+    value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
+    value = false if value == 'false'
+
+    Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
+    GOT_KEYS << [key, value, tokens]
+
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    elsif value == 'nil'
+      nil
+    else
+      value
+    end
+  end
+
+  def self.with_config
+      saved_config = {}
+      CACHE.each do |k,v|
+        saved_config[k] = v.dup
+      end
+      saved_got_keys = GOT_KEYS.dup
+    begin
+      yield
+    ensure
+      CACHE.replace(saved_config)
+      GOT_KEYS.replace(saved_got_keys)
+    end
+  end
+
+  def self.process_config(config)
+    if Path.is_filename?(config) && File.exist?(config)
+      Scout::Config.load_file(config)
+    elsif Scout.etc.config_profile[config].exists?
+      Scout::Config.load_file(Scout.etc.config_profile[config].find)
+    else
+      key, value, *tokens = config.split(/\s/)
+      tokens = tokens.collect do |tok|
+        tok, _sep, prio = tok.partition("::")
+        prio = "0" if prio.nil? or prio.empty?
+        [tok, prio] * "::"
+      end
+      Scout::Config.set({key => value}, *tokens)
+    end
+  end
+
+
+  self.load_config
+end

data/lib/scout/exceptions.rb

@@ -0,0 +1,77 @@
+class ScoutDeprecated < StandardError; end
+class ScoutException < StandardError; end
+
+class FieldNotFoundError < StandardError;end
+
+class TryAgain < StandardError; end
+class StopInsist < Exception
+  attr_accessor :exception
+  def initialize(exception)
+    @exception = exception
+  end
+end
+
+class Aborted < StandardError; end
+
+class ParameterException < ScoutException; end
+class MissingParameterException < ParameterException
+  def initialize(parameter)
+    super("Missing parameter '#{parameter}'")
+  end
+end
+class ProcessFailed < StandardError; 
+  attr_accessor :pid, :msg
+  def initialize(pid = Process.pid, msg = nil)
+    @pid = pid
+    @msg = msg
+    if @pid
+      if @msg
+        message = "Process #{@pid} failed - #{@msg}"
+      else
+        message = "Process #{@pid} failed"
+      end
+    else
+      message = "Failed to run #{@msg}"
+    end
+    super(message)
+  end
+end
+
+class ConcurrentStreamProcessFailed < ProcessFailed
+  attr_accessor :concurrent_stream
+  def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)
+    super(pid, msg)
+    @concurrent_stream = concurrent_stream
+  end
+end
+
+class OpenURLError < StandardError; end
+
+class DontClose < Exception
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
+
+class KeepLocked < Exception
+  attr_accessor :payload
+  def initialize(payload)
+    @payload = payload
+  end
+end
+
+class KeepBar < Exception
+  attr_accessor :payload
+  def initialize(payload)
+    @payload = payload
+  end
+end
+
+class LockInterrupted < TryAgain; end
+
+class ClosedStream < StandardError; end
+
+class ResourceNotFound < ScoutException; end
+

data/lib/scout/indiferent_hash/case_insensitive.rb

@@ -0,0 +1,30 @@
+module CaseInsensitiveHash
+
+  def self.setup(hash)
+    hash.extend CaseInsensitiveHash
+  end
+
+  def downcase_keys
+    @downcase_keys ||= begin
+                         down = {} 
+                         keys.collect{|key| 
+                           down[key.to_s.downcase] = key 
+                         }
+                         down
+                       end
+  end
+
+  def [](key, *rest)
+    value = super(key, *rest)
+    return value unless value.nil?
+    key_downcase = key.to_s.downcase
+    super(downcase_keys[key_downcase])
+  end
+
+  def values_at(*keys)
+    keys.collect do |key|
+      self[key]
+    end
+  end
+  
+end

data/lib/scout/indiferent_hash/options.rb

@@ -0,0 +1,115 @@
+module IndiferentHash
+  def self.add_defaults(options, defaults = {})
+    options = string2hash options if String === options
+    IndiferentHash.setup(options)
+
+    defaults = string2hash defaults if String === defaults
+
+    defaults.each do |key, value|
+      next if options.include?(key)
+
+      options[key] = value 
+    end
+
+    options
+  end
+
+  def self.process_options(hash, *keys)
+    IndiferentHash.setup(hash)
+
+    defaults = keys.pop if Hash === keys.last
+    hash = IndiferentHash.add_defaults hash, defaults if defaults
+
+    if keys.length == 1
+      hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
+    else
+      keys.collect do |key| hash.include?(key.to_sym) ? hash.delete(key.to_sym) : hash.delete(key.to_s) end
+    end
+  end
+
+  def self.pull_keys(hash, prefix)
+    new = {}
+    prefix = prefix.to_s
+    hash.keys.each do |key|
+      if key.to_s =~ /#{ prefix }_(.*)/
+        case
+        when String === key
+          new[$1] = hash.delete key
+        when Symbol === key
+          new[$1.to_sym] = hash.delete key
+        end
+      else
+        if key.to_s == prefix.to_s
+          new[key] = hash.delete key
+        end
+      end
+    end
+
+    IndiferentHash.setup(new)
+  end
+
+  def self.zip2hash(list1, list2)
+    hash = {}
+    list1.each_with_index do |e,i|
+      hash[e] = list2[i]
+    end
+    IndiferentHash.setup(hash)
+  end
+
+  def self.positional2hash(keys, *values)
+    if Hash === values.last
+      extra = values.pop
+      inputs = IndiferentHash.zip2hash(keys, values)
+      inputs.delete_if{|k,v| v.nil? or (String === v and v.empty?)}
+      inputs = IndiferentHash.add_defaults inputs, extra
+      inputs.delete_if{|k,v| not keys.include?(k) and not (Symbol === k ? keys.include?(k.to_s) : keys.include?(k.to_sym))}
+      inputs
+    else
+      IndiferentHash.zip2hash(keys, values)
+    end
+  end
+
+  def self.array2hash(array, default = nil)
+    hash = {}
+    array.each do |key, value|
+      value = default.dup if value.nil? and not default.nil?
+      hash[key] = value
+    end
+    IndiferentHash.setup(hash)
+  end
+
+  def self.process_to_hash(list)
+    result = yield list
+    zip2hash(list, result)
+  end
+
+  def self.hash2string(hash)
+    hash.sort_by{|k,v| k.to_s}.collect{|k,v| 
+      next unless %w(Symbol String Float Fixnum Integer Numeric TrueClass FalseClass Module Class Object).include? v.class.to_s
+      [ Symbol === k ? ":" << k.to_s : k.to_s.chomp,
+        Symbol === v ? ":" << v.to_s : v.to_s.chomp] * "="
+    }.compact * "#"
+  end
+
+  def self.string2hash(string)
+    options = {}
+
+    string.split('#').each do |str|
+      key, _, value = str.partition "="
+
+      key = key[1..-1].to_sym if key[0] == ":"
+
+      options[key] = true and next if value.empty?
+      options[key] = value[1..-1].to_sym and next if value[0] == ":"
+      options[key] = Regexp.new(/#{value[1..-2]}/) and next if value[0] == "/" and value[-1] == "/"
+      options[key] = value[1..-2] and next if value =~ /^['"].*['"]$/
+      options[key] = value.to_i and next if value =~ /^\d+$/
+      options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
+      options[key] = true and next if value == "true"
+      options[key] = false and next if value == "false"
+      options[key] = value
+    end
+
+    IndiferentHash.setup(options)
+  end
+end