scout-gear 9.1.0 → 10.1.0

data/lib/scout/concurrent_stream.rb

@@ -1,284 +0,0 @@
-require_relative 'indiferent_hash'
-
-module AbortedStream
-  attr_accessor :exception
-  def self.setup(obj, exception = nil)
-    obj.extend AbortedStream
-    obj.exception = exception
-  end
-end
-
-module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
-
-  def self.setup(stream, options = {}, &block)
-    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
-    stream.extend ConcurrentStream unless ConcurrentStream === stream
-
-    stream.threads ||= []
-    stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
-    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
-    stream.autojoin = autojoin unless autojoin.nil?
-    stream.no_fail = no_fail unless no_fail.nil?
-    stream.std_err = ""
-
-    stream.next = next_stream unless next_stream.nil?
-    stream.pair = pair unless pair.nil?
-
-    callback = block if block_given?
-    if callback
-      if stream.callback
-        old_callback = stream.callback
-        stream.callback = Proc.new do
-          old_callback.call
-          callback.call
-        end
-      else
-        stream.callback = callback
-      end
-    end
-
-    if abort_callback
-      if stream.abort_callback
-        old_abort_callback = stream.abort_callback
-        stream.abort_callback = Proc.new do
-          old_abort_callback.call
-          abort_callback.call
-        end
-      else
-        stream.abort_callback = abort_callback
-      end
-    end
-
-    stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
-
-    stream.lock = lock unless lock.nil?
-
-    stream.aborted = false
-
-    stream
-  end
-
-  def annotate(stream)
-    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
-    stream
-  end
-
-  def clear
-    @threads = @pids = @callback = @abort_callback = @joined = nil
-  end
-
-  def joined?
-    @joined
-  end
-
-  def aborted?
-    @aborted
-  end
-
-  def join_threads
-    if @threads
-      @threads.each do |t|
-        next if t == Thread.current
-        begin
-          t.join
-          if Process::Status === t.value
-            if ! (t.value.success? || no_fail)
-
-              if log
-                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
-              else
-                msg = "Error joining #{self.filename || self.inspect}"
-              end
-
-              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
-            end
-          end
-        rescue Exception
-          if no_fail
-            Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
-          else
-            Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
-            stream_raise_exception $!
-          end
-        end
-      end
-    end
-    @threads = []
-  end
-
-  def join_pids
-    if @pids and @pids.any?
-      @pids.each do |pid|
-        begin
-          Process.waitpid(pid, Process::WUNTRACED)
-          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
-        rescue Errno::ECHILD
-        end
-      end
-      @pids = []
-    end
-  end
-
-  def join_callback
-    if @callback and not joined?
-      begin
-        @callback.call
-      ensure
-        @callback = nil
-      end
-    end
-  end
-
-  def join
-    begin
-      join_threads
-      join_pids
-      raise stream_exception if stream_exception
-      join_callback
-      close unless closed?
-    ensure
-      @joined = true
-      begin
-        lock.unlock if lock && lock.locked?
-      rescue
-        Log.exception $!
-      end
-      raise stream_exception if stream_exception
-    end
-  end
-
-  def abort_threads(exception = nil)
-    return unless @threads and @threads.any?
-    name = Log.fingerprint(Thread.current)
-    name += " - file:#{filename}" if filename
-    Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
-
-    threads = @threads.dup
-    @threads.clear
-    threads.each do |t|
-      next if t == Thread.current
-      next if t["aborted"]
-      t["aborted"] = true
-      exception = exception.nil? ? Aborted.new : exception
-      Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
-      t.raise(exception)
-      t.join
-    end
-  end
-
-  def abort_pids
-    @pids.each do |pid|
-      begin
-        Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
-        Process.kill :INT, pid
-      rescue Errno::ESRCH
-      end
-    end if @pids
-    @pids = []
-  end
-
-  def abort(exception = nil)
-    self.stream_exception ||= exception
-    if @aborted
-      Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
-      return
-    else
-      Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
-    end
-    AbortedStream.setup(self, exception)
-    @aborted = true
-    begin
-      @abort_callback.call exception if @abort_callback
-
-      abort_threads(exception)
-      abort_pids
-
-      @callback = nil
-      @abort_callback = nil
-
-      if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
-        Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
-        @pair.abort exception
-      end
-    ensure
-      close unless closed?
-
-      if lock and lock.locked?
-        lock.unlock
-      end
-    end
-  end
-
-  def close(*args)
-    if autojoin
-      begin
-        super(*args)
-      rescue
-        self.abort
-        self.join
-        stream_raise_exception $!
-      ensure
-        self.join if ! @stream_exception && (self.closed? || self.eof?)
-      end
-    else
-      super(*args)
-    end
-  end
-
-  def read(*args)
-    begin
-      super(*args)
-    rescue Exception
-      @stream_exception ||= $!
-      raise @stream_exception
-    ensure
-      if ! @stream_exception && autojoin && ! closed?
-        begin
-          done = eof?
-        rescue Exception
-          self.abort($!)
-          raise $!
-        end
-        close if done
-      end
-    end
-  end
-
-  def add_callback(&block)
-    old_callback = callback
-    @callback = Proc.new do
-      old_callback.call if old_callback
-      block.call
-    end
-  end
-
-  def stream_raise_exception(exception)
-    self.stream_exception = exception
-    threads.each do |thread|
-      thread.raise exception
-    end
-    self.abort
-  end
-
-  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
-    ConcurrentStream.setup(stream, **kwargs)
-    begin
-      begin
-        yield
-      ensure
-        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
-        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
-      end
-    rescue Aborted
-      Log.low "Aborted #{message}: #{$!.message}"
-      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
-      raise $!
-    rescue Exception
-      Log.low "Exception #{message}: #{$!.message}"
-      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
-      raise $!
-    end
-  end
-end

data/lib/scout/config.rb

@@ -1,168 +0,0 @@
-require_relative 'path'
-require_relative 'resource'
-require_relative 'resource/scout'
-
-module Scout::Config
-
-  CACHE ||= IndiferentHash.setup({})
-
-  GOT_KEYS=[]
-
-  def self.add_entry(key, value, tokens)
-    tokens = [tokens] unless Array === tokens
-    tokens << "key:#{key}" unless tokens.include?("key:#{key}")
-    CACHE[key.to_s] ||= [] 
-    CACHE[key.to_s] << [tokens, value]
-  end
-
-  def self.load_file(file)
-    Log.debug "Loading config file: #{ file }"
-    TSV.traverse file, :type => :array do |line|
-      next if line =~ /^#/
-      key, value, *tokens = line.strip.split(/\s/)
-
-      self.add_entry(key, value, tokens) if key
-    end
-  end
-
-  def self.load_config
-    Scout.etc.config.find_all.reverse.each do |file|
-      self.load_file(file)
-    end
-  end
-
-  def self.set(values, *tokens)
-    if not Hash === values
-      values = {values => tokens.shift}
-    end
-
-    values.each do |key,value|
-      add_entry key, value, tokens
-    end
-  end
-
-  def self.token_priority(token)
-    token, _sep, priority = token.to_s.partition("::")
-
-    if priority.nil? || priority.empty?
-      type, _sep, rest = token.partition(":")
-      priority = case type
-                 when "workflow"
-                   4
-                 when "task"
-                   3
-                 when "file"
-                   2
-                 when "line"
-                   1
-                 when "key"
-                   20
-                 else
-                   10
-                 end
-    else
-      priority = priority.to_i
-    end
-
-    [token, priority]
-  end
-
-  def self.match(entries, give_token)
-    priorities = {}
-    entries.each do |tokens, value|
-      best_prio = nil
-      tokens = [tokens] unless Array === tokens
-      tokens.each do |tok|
-        tok, prio = token_priority tok
-        next unless tok == give_token
-
-        best_prio = prio if best_prio.nil? or best_prio > prio
-        next if prio > best_prio
-
-        priorities[prio] ||= []
-        priorities[prio].unshift value
-      end
-    end if entries
-    priorities
-  end
-
-  # For equal priorities the matching prioritizes tokens ealier in the list
-  def self.get(key, *tokens)
-    options = tokens.pop if Hash === tokens.last
-    default = options.nil? ? nil : options[:default]
-
-    tokens = ["key:" + key] if tokens.empty?
-
-    tokens = tokens.flatten
-    file, _sep, line = caller.reject{|l| 
-      l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
-        l =~ /rbbt\/resource\/path\.rb/ or
-        l =~ /rbbt\/util\/misc\.rb/ or
-        l =~ /accessor\.rb/ or
-        l =~ /progress-monitor\.rb/ 
-    }.first.partition(":")
-
-    File.expand_path(file)
-
-    tokens << ("file:" << file)
-    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
-
-    entries = CACHE[key.to_s]
-    priorities = {}
-    tokens.each do |token|
-      token_prio = match entries, token.to_s
-      token_prio.each do |prio, values|
-        priorities[prio] ||= []
-        priorities[prio].concat(values)
-      end
-    end
-
-    value = priorities.empty? ? default : priorities.collect{|p| p }.sort_by{|p,v| p}.first.last.first
-    value = false if value == 'false'
-
-    Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
-    GOT_KEYS << [key, value, tokens]
-
-    if String === value && m = value.match(/^env:(.*)/)
-      variable = m.captures.first
-      ENV[variable]
-    elsif value == 'nil'
-      nil
-    else
-      value
-    end
-  end
-
-  def self.with_config
-      saved_config = {}
-      CACHE.each do |k,v|
-        saved_config[k] = v.dup
-      end
-      saved_got_keys = GOT_KEYS.dup
-    begin
-      yield
-    ensure
-      CACHE.replace(saved_config)
-      GOT_KEYS.replace(saved_got_keys)
-    end
-  end
-
-  def self.process_config(config)
-    if Path.is_filename?(config) && File.exist?(config)
-      Scout::Config.load_file(config)
-    elsif Scout.etc.config_profile[config].exists?
-      Scout::Config.load_file(Scout.etc.config_profile[config].find)
-    else
-      key, value, *tokens = config.split(/\s/)
-      tokens = tokens.collect do |tok|
-        tok, _sep, prio = tok.partition("::")
-        prio = "0" if prio.nil? or prio.empty?
-        [tok, prio] * "::"
-      end
-      Scout::Config.set({key => value}, *tokens)
-    end
-  end
-
-
-  self.load_config
-end

data/lib/scout/exceptions.rb

@@ -1,151 +0,0 @@
-class ScoutDeprecated < StandardError; end
-class ScoutException < StandardError; end
-
-class FieldNotFoundError < StandardError;end
-
-class TryAgain < StandardError; end
-class StopInsist < Exception
-  attr_accessor :exception
-  def initialize(exception)
-    @exception = exception
-  end
-end
-
-class Aborted < StandardError; end
-
-class ParameterException < ScoutException; end
-class MissingParameterException < ParameterException
-  def initialize(parameter)
-    super("Missing parameter '#{parameter}'")
-  end
-end
-class ProcessFailed < StandardError; 
-  attr_accessor :pid, :msg
-  def initialize(pid = Process.pid, msg = nil)
-    @pid = pid
-    @msg = msg
-    if @pid
-      if @msg
-        message = "Process #{@pid} failed - #{@msg}"
-      else
-        message = "Process #{@pid} failed"
-      end
-    else
-      message = "Failed to run #{@msg}"
-    end
-    super(message)
-  end
-end
-
-class ConcurrentStreamProcessFailed < ProcessFailed
-  attr_accessor :concurrent_stream
-  def initialize(pid = Process.pid, msg = nil, concurrent_stream = nil)
-    super(pid, msg)
-    @concurrent_stream = concurrent_stream
-  end
-end
-
-class OpenURLError < StandardError; end
-
-class DontClose < Exception
-  attr_accessor :payload
-  def initialize(payload = nil)
-    @payload = payload
-  end
-end
-
-
-class KeepLocked < Exception
-  attr_accessor :payload
-  def initialize(payload)
-    @payload = payload
-  end
-end
-
-class KeepBar < Exception
-  attr_accessor :payload
-  def initialize(payload)
-    @payload = payload
-  end
-end
-
-class LockInterrupted < TryAgain; end
-
-class ClosedStream < StandardError; end
-
-class DoneProcessing < Exception
-  attr_accessor :pid
-  def initialize(pid = Process.pid)
-    @pid = pid
-  end
-
-  def message
-    "Done processing pid #{pid}"
-  end
-end
-
-class WorkerException < ScoutException
-  attr_accessor :worker_exception, :pid
-  def initialize(worker_exception, pid)
-    @worker_exception = worker_exception
-    @pid = pid
-  end
-end
-
-class SemaphoreInterrupted < TryAgain; end
-
-#class OpenGzipError < StandardError; end
-#
-#
-#class TryThis < StandardError
-#  attr_accessor :payload
-#  def initialize(payload = nil)
-#    @payload = payload
-#  end
-#end
-#
-#class SemaphoreInterrupted < TryAgain; end
-#
-#class RemoteServerError < StandardError; end
-#
-#class DependencyError < Aborted
-#  def initialize(msg)
-#    if defined? Step and Step === msg
-#      step = msg
-#      new_msg = [step.path, step.messages.last] * ": "
-#      super(new_msg)
-#    else
-#      super(msg)
-#    end
-#  end
-#end
-#
-#class DependencyScoutException < ScoutException
-#  def initialize(msg)
-#    if defined? Step and Step === msg
-#      step = msg
-#
-#      new_msg = nil
-#      new_msg = [step.path, step.messages.last] * ": "
-#
-#      super(new_msg)
-#    else
-#      super(msg)
-#    end
-#  end
-#end
-#
-#
-#
-#
-class ResourceNotFound < ScoutException; end
-
-class SSHProcessFailed < StandardError
-  attr_accessor :host, :cmd
-  def initialize(host, cmd)
-    @host = host
-    @cmd = cmd
-    message = "SSH server #{host} failed cmd '#{cmd}'" 
-    super(message)
-  end
-end

data/lib/scout/indiferent_hash/case_insensitive.rb

@@ -1,30 +0,0 @@
-module CaseInsensitiveHash
-
-  def self.setup(hash)
-    hash.extend CaseInsensitiveHash
-  end
-
-  def downcase_keys
-    @downcase_keys ||= begin
-                         down = {} 
-                         keys.collect{|key| 
-                           down[key.to_s.downcase] = key 
-                         }
-                         down
-                       end
-  end
-
-  def [](key, *rest)
-    value = super(key, *rest)
-    return value unless value.nil?
-    key_downcase = key.to_s.downcase
-    super(downcase_keys[key_downcase])
-  end
-
-  def values_at(*keys)
-    keys.collect do |key|
-      self[key]
-    end
-  end
-  
-end