scout-gear 1.2.0 → 5.1.1

data/lib/scout/concurrent_stream.rb

@@ -0,0 +1,259 @@
+require_relative 'indiferent_hash'
+
+module AbortedStream
+  attr_accessor :exception
+  def self.setup(obj, exception = nil)
+    obj.extend AbortedStream
+    obj.exception = exception
+  end
+end
+
+module ConcurrentStream
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
+
+  def self.setup(stream, options = {}, &block)
+    
+    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+    stream.autojoin = autojoin unless autojoin.nil?
+    stream.no_fail = no_fail unless no_fail.nil?
+    stream.std_err = ""
+
+    stream.pair = pair unless pair.nil?
+
+    callback = block if block_given?
+    if callback
+      if stream.callback
+        old_callback = stream.callback
+        stream.callback = Proc.new do
+          old_callback.call
+          callback.call
+        end
+      else
+        stream.callback = callback 
+      end
+    end
+
+    if abort_callback
+      if stream.abort_callback
+        old_abort_callback = stream.abort_callback
+        stream.abort_callback = Proc.new do
+          old_abort_callback.call
+          abort_callback.call
+        end
+      else
+        stream.abort_callback = abort_callback 
+      end
+    end
+
+    stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
+
+    stream.lockfile = lockfile unless lockfile.nil?
+
+    stream.aborted = false
+
+    stream
+  end
+
+  def annotate(stream)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
+    stream
+  end
+
+  def clear
+    @threads = @pids = @callback = @abort_callback = @joined = nil
+  end
+
+  def joined?
+    @joined
+  end
+
+  def aborted?
+    @aborted
+  end
+
+  def join_threads
+    if @threads
+      @threads.each do |t| 
+        next if t == Thread.current
+        begin
+          t.join
+          if Process::Status === t.value
+            if ! (t.value.success? || no_fail)
+
+              if log
+                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+              else
+                msg = "Error joining #{self.filename || self.inspect}"
+              end
+
+              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
+            end
+          end
+        rescue Exception
+          if no_fail
+            Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
+          else
+            Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
+            stream_raise_exception $! 
+          end
+        end
+      end
+    end
+    @threads = []
+  end
+
+  def join_pids
+    if @pids and @pids.any?
+      @pids.each do |pid| 
+        begin
+          Process.waitpid(pid, Process::WUNTRACED)
+          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
+        rescue Errno::ECHILD
+        end
+      end 
+      @pids = []
+    end
+  end
+
+  def join_callback
+    if @callback and not joined?
+      begin
+        @callback.call
+      ensure
+        @callback = nil
+      end
+    end
+  end
+
+  def join
+    begin
+      join_threads
+      join_pids
+      join_callback
+      close unless closed?
+    ensure
+      @joined = true
+      lockfile.unlock if lockfile and lockfile.locked?
+      raise stream_exception if stream_exception
+    end
+  end
+
+  def abort_threads(exception = nil)
+    return unless @threads and @threads.any?
+    name = Log.fingerprint(Thread.current)
+    name += " - file:#{filename}" if filename
+    Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
+
+    @threads.each do |t| 
+      next if t == Thread.current
+      Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
+      t.raise((exception.nil? ? Aborted.new : exception))
+    end 
+
+    @threads.each do |t|
+      next if t == Thread.current
+      begin
+        t.join unless t == Thread.current
+      rescue Aborted
+      rescue Exception
+        Log.debug "Thread (#{name}) exception: #{$!.message}"
+      end
+    end
+  end
+
+  def abort_pids
+    @pids.each do |pid|
+      begin 
+        Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
+        Process.kill :INT, pid 
+      rescue Errno::ESRCH
+      end
+    end if @pids
+    @pids = []
+  end
+
+  def abort(exception = nil)
+    self.stream_exception ||= exception
+    if @aborted
+      Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
+      return
+    else
+      Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
+    end
+    AbortedStream.setup(self, exception)
+    @aborted = true 
+    begin
+      @abort_callback.call exception if @abort_callback
+
+      abort_threads(exception)
+      abort_pids
+
+      @callback = nil
+      @abort_callback = nil
+
+      if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
+        Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
+        @pair.abort exception
+      end
+    ensure
+      close unless closed?
+
+      if lockfile and lockfile.locked?
+        lockfile.unlock 
+      end
+    end
+  end
+
+  def close(*args)
+    if autojoin
+      begin
+        super(*args)
+      rescue
+        Log.exception $!
+        self.abort
+        self.join 
+        stream_raise_exception $!
+      ensure
+        self.join if self.closed? or self.eof? 
+      end
+    else
+      super(*args)
+    end
+  end
+
+  def read(*args)
+    begin
+      super(*args)
+    ensure
+      begin
+        close unless closed?
+      rescue Exception
+        raise $! if ConcurrentStreamProcessFailed === $!
+      end if autojoin && ! closed? && eof?
+    end
+  end
+
+  def add_callback(&block)
+    old_callback = callback
+    @callback = Proc.new do 
+      old_callback.call if old_callback
+      block.call
+    end
+  end
+
+  def stream_raise_exception(exception)
+    threads.each do |thread|
+      thread.raise exception
+    end
+    self.stream_exception = exception
+
+    self.abort
+  end
+
+end

data/lib/scout/exceptions.rb

@@ -13,6 +13,12 @@ end
 
 class Aborted < StandardError; end
 
+class ParameterException < ScoutException; end
+class MissingParameterException < ParameterException
+  def initialize(parameter)
+    super("Missing parameter '#{parameter}'")
+  end
+end
 class ProcessFailed < StandardError; 
   attr_accessor :pid, :msg
   def initialize(pid = Process.pid, msg = nil)
@@ -41,7 +47,13 @@ end
 
 class OpenURLError < StandardError; end
 
-class DontClose < Exception; end
+class DontClose < Exception
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
 
 class KeepLocked < Exception
   attr_accessor :payload
@@ -57,13 +69,9 @@ class KeepBar < Exception
   end
 end
 
-#class ParameterException < ScoutException; end
+class LockInterrupted < TryAgain; end
+
 #
-#class MissingParameterException < ParameterException
-#  def initialize(parameter)
-#    super("Missing parameter '#{parameter}'")
-#  end
-#end
 #class ClosedStream < StandardError; end
 #class OpenGzipError < StandardError; end
 #
@@ -76,7 +84,6 @@ end
 #end
 #
 #class SemaphoreInterrupted < TryAgain; end
-#class LockInterrupted < TryAgain; end
 #
 #class RemoteServerError < StandardError; end
 #

data/lib/scout/indiferent_hash/options.rb

@@ -1,32 +1,24 @@
 module IndiferentHash
   def self.add_defaults(options, defaults = {})
-    options ||= {}
+    options = string2hash options if String === options
     IndiferentHash.setup(options)
-    case
-    when Hash === options
-      new_options = options.dup
-    when String === options
-      new_options = string2hash options
-    else
-      raise "Format of '#{options.inspect}' not understood. It should be a hash"
-    end
+
+    defaults = string2hash defaults if String === defaults
 
     defaults.each do |key, value|
       next if options.include? key
 
-      new_options[key] = value 
+      options[key] = value 
     end
 
-    new_options
-
-    options.replace new_options
+    options
   end
 
   def self.process_options(hash, *keys)
     IndiferentHash.setup(hash)
 
     defaults = keys.pop if Hash === keys.last
-    hahs = IndiferentHash.add_defaults hash, defaults if defaults
+    hash = IndiferentHash.add_defaults hash, defaults if defaults
 
     if keys.length == 1
       hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
@@ -102,7 +94,7 @@ module IndiferentHash
     options = {}
 
     string.split('#').each do |str|
-      key, sep, value = str.partition "="
+      key, _, value = str.partition "="
 
       key = key[1..-1].to_sym if key[0] == ":"
 
@@ -114,17 +106,7 @@ module IndiferentHash
       options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
       options[key] = true and next if value == "true"
       options[key] = false and next if value == "false"
-      options[key] = value and next 
-
-      options[key] = begin
-                       saved_safe = $SAFE
-                       $SAFE = 0
-                       eval(value)
-                     rescue Exception
-                       value
-                     ensure
-                       $SAFE = saved_safe
-                     end
+      options[key] = value
     end
 
     IndiferentHash.setup(options)

data/lib/scout/indiferent_hash.rb

@@ -75,33 +75,3 @@ module IndiferentHash
   end
 end
 
-module CaseInsensitiveHash
-
-  def self.setup(hash)
-    hash.extend CaseInsensitiveHash
-  end
-
-  def downcase_keys
-    @downcase_keys ||= begin
-                         down = {} 
-                         keys.collect{|key| 
-                           down[key.to_s.downcase] = key 
-                         }
-                         down
-                       end
-  end
-
-  def [](key, *rest)
-    value = super(key, *rest)
-    return value unless value.nil?
-    key_downcase = key.to_s.downcase
-    super(downcase_keys[key_downcase])
-  end
-
-  def values_at(*keys)
-    keys.collect do |key|
-      self[key]
-    end
-  end
-  
-end

data/lib/scout/log/color.rb

@@ -63,7 +63,7 @@ module Colorize
     if percent
       array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
     else
-      array = array.collect{|v| n = v.to_f; } 
+      array = array.collect{|v| v.to_f } 
     end
     max = array.max
     min = array.min
@@ -103,7 +103,7 @@ module Colorize
 
     value_color = Hash[*array.uniq.zip(all_colors).flatten]
 
-    value_color.values_at *array
+    value_color.values_at(*array)
   end
 
   def self.tsv(tsv, options = {})

data/lib/scout/log/fingerprint.rb

@@ -19,8 +19,12 @@ module Log
       else 
         "'" << obj << "'"
       end
+    when ConcurrentStream
+      name = obj.inspect + " " + obj.object_id.to_s
+      name += " #{obj.filename}" if obj.filename
+      name
     when IO
-      (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect + rand(100000)) + ">" : obj.inspect
+      (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect + rand(100000)) + ">" : obj.inspect + " " + obj.object_id.to_s
     when File
       "<File:" + obj.path + ">"
     when Array
@@ -52,6 +56,12 @@ module Log
       else
         "%.6f" % obj
       end
+    when Thread
+      if obj["name"]
+        obj["name"]
+      else
+        obj.inspect
+      end
     else
       obj.to_s
     end

data/lib/scout/log/progress/report.rb

@@ -169,7 +169,6 @@ module Log
           bar = bars.sort_by{|b| b.depth }.first
           print(io, Log.color(:magenta ,bar.report_msg) << "\n") 
         else
-          length = Log::ProgressBar.cleanup_bars
           print(io, Log.color(:magenta, "···Progress\n"))
           bars.sort_by{|b| b.depth }.reverse.each do |bar|
             if SILENCED.include? bar

data/lib/scout/log/progress/util.rb

@@ -91,7 +91,7 @@ module Log
         error = true
         raise $!
       ensure
-        remove_bar(bar, error) if bar
+        remove_bar(bar, error) if bar && ! keep
       end
     end
   end

data/lib/scout/log/progress.rb

@@ -17,12 +17,12 @@ module Log
       attr_accessor :default_file
     end
 
-    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
+    attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback, :severity
 
     def initialize(max = nil, options = {})
-      depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback = 
-        IndiferentHash.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback,
-        :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
+      depth, desc, file, bytes, frequency, process, callback = 
+        IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
+        :depth => 0, :frequency => 2
 
       @max = max
       @ticks = 0

data/lib/scout/log.rb

@@ -14,8 +14,8 @@ module Log
                      end
 
   class << self
-    attr_accessor :logfile, :severity
-    attr_writer :tty_size
+    attr_accessor :severity
+    attr_writer :tty_size, :logfile
   end
 
 
@@ -360,3 +360,11 @@ def eef(obj=nil, file = $stdout)
   Log.log_obj_fingerprint(obj, :error, file)
 end
 
+def sss(level, &block)
+  if block_given?
+    Log.with_severity level, &block
+  else
+    Log.severity = level
+  end
+end
+

data/lib/scout/meta_extension.rb

@@ -11,7 +11,13 @@ module MetaExtension
       end
     end
 
-    meta.define_method(:setup) do |obj,*rest|
+    meta.define_method(:setup) do |*args,&block|
+      if block_given?
+        obj, rest = block, args
+      else
+        obj, *rest = args
+      end
+      obj = block if obj.nil?
       obj.extend base
       attrs = self.class_variable_get("@@extension_attrs")
 
@@ -33,6 +39,14 @@ module MetaExtension
       obj
     end
 
+    base.define_method(:extension_attr_hash) do 
+      attr_hash = {}
+      meta.class_variable_get("@@extension_attrs").each do |name|
+        attr_hash[name] = self.instance_variable_get("@#{name}")
+      end
+      attr_hash
+    end
+
     base.define_method(:annotate) do |other|
       attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
         self.instance_variable_get("@#{a}")

data/lib/scout/misc/digest.rb

@@ -0,0 +1,56 @@
+module Misc
+  def self.digest_str(obj)
+    if obj.respond_to?(:digest_str)
+      obj.digest_str
+    else
+      case obj
+      when String
+        #'\'' << obj << '\''
+        if Path === obj || ! Open.exists?(obj)
+          '\'' << obj << '\''
+        else
+          Misc.file_md5(obj)
+        end
+      when Integer, Symbol
+        obj.to_s
+      when Array
+        '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+      when Hash
+        '{' << obj.inject(""){|acc,p| s = Misc.digest_str(p.first) << "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
+      when Integer
+        obj.to_s
+      when Float
+        if obj % 1 == 0
+          obj.to_i
+        elsif obj.abs > 10
+          "%.1f" % obj
+        elsif obj.abs > 1
+          "%.3f" % obj
+        else
+          "%.6f" % obj
+        end
+      when TrueClass
+        "true"
+      when FalseClass
+        "false"
+      else
+        obj.inspect
+      end
+    end
+  end
+
+  def self.digest(obj)
+    str = Misc.digest_str(obj)
+    hash = Digest::MD5.hexdigest(str)
+    Log.debug "Digest #{hash} - #{str}"
+    hash
+  end
+
+  def self.file_md5(file)
+    file = file.find if Path === file
+    #md5file = file + '.md5'
+    Persist.persist("MD5:#{file}", :string) do
+      Digest::MD5.file(file).hexdigest
+    end
+  end
+end

data/lib/scout/misc/filesystem.rb

@@ -0,0 +1,26 @@
+module Misc
+  def self.in_dir(dir)
+    old_pwd = FileUtils.pwd
+    begin
+      FileUtils.mkdir_p dir unless File.exist?(dir)
+      FileUtils.cd dir
+      yield
+    ensure
+      FileUtils.cd old_pwd
+    end
+  end
+
+  def self.path_relative_to(basedir, path)
+    path = File.expand_path(path) unless path.slice(0,1) == "/"
+    basedir = File.expand_path(basedir) unless basedir.slice(0,1) == "/"
+
+    basedir += "/" unless basedir.slice(-2,-1) == "/"
+
+    if path.index(basedir) == 0
+      return path[basedir.length..-1]
+    else
+      return nil
+    end
+  end
+
+end