scout-gear 5.2.0 → 7.1.0

data/VERSION

@@ -1 +1 @@
-5.2.0
+7.1.0

data/lib/scout/exceptions.rb

@@ -71,8 +71,28 @@ end
 
 class LockInterrupted < TryAgain; end
 
-#
-#class ClosedStream < StandardError; end
+class ClosedStream < StandardError; end
+
+class DoneProcessing < Exception
+  attr_accessor :pid
+  def initialize(pid = Process.pid)
+    @pid = pid
+  end
+
+  def message
+    "Done processing pid #{pid}"
+  end
+end
+
+class WorkerException < ScoutException
+  attr_accessor :exception, :pid
+  def initialize(exception, pid)
+    @exception = exception
+    @pid = pid
+  end
+end
+
+
 #class OpenGzipError < StandardError; end
 #
 #

data/lib/scout/log/color.rb

@@ -2,6 +2,7 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
+require 'colorist'
 
 module Colorize
   def self.colors=(colors)
@@ -9,7 +10,13 @@ module Colorize
   end
 
   def self.colors
-    @colors ||= IndiferentHash.setup({green: "#00cd00" , red: "#cd0000" , yellow: "#ffd700" })
+    @colors ||= IndiferentHash.setup(Hash[<<-EOF.split("\n").collect{|l| l.split(" ")}])
+green #00cd00 
+red #cd0000 
+yellow #ffd700
+blue #0000cd
+path blue
+EOF
   end
 
   def self.diverging_colors=(colors)
@@ -33,7 +40,6 @@ module Colorize
     EOF
   end
 
-
   def self.from_name(color)
     return color if color =~ /^#?[0-9A-F]+$/i
     return colors[color.to_s] if colors.include?(color.to_s)
@@ -52,7 +58,7 @@ module Colorize
     when 'blue'
       colors["RoyalBlue"] 
     else
-      colors[color.to_s] || color
+      colors[color.to_s] || color.to_s
     end
   end
 
@@ -134,8 +140,22 @@ module Log
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
   SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
+  CONCEPT_COLORS = IndiferentHash.setup({
+    :title => magenta,
+    :path => blue,
+    :input => cyan,
+    :value => green,
+    :integer => green,
+    :negative => red,
+    :float => green,
+    :waiting => yellow,
+    :started => cyan,
+    :start => cyan,
+    :done => green,
+    :error => red,
+  })
   HIGHLIGHT = "\033[1m"
-
+  
   def self.uncolor(str)
     "" << Term::ANSIColor.uncolor(str)
   end
@@ -144,15 +164,46 @@ module Log
     reset
   end
 
-  def self.color(severity, str = nil, reset = false)
+  def self.color(color, str = nil, reset = false)
     return str.dup || "" if nocolor 
-    color = reset ? Term::ANSIColor.reset : ""
-    color << SEVERITY_COLOR[severity] if Integer === severity
-    color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
+
+    if (color == :integer || color == :float) && Numeric === str
+      color = if str < 0
+                :red
+              elsif str > 1
+                :cyan
+              else
+                :green
+              end
+    end
+
+    if color == :status 
+      color = case str.to_sym
+              when :done
+                :green
+              when :error, :aborted
+                :red
+              when :waiting, :queued
+                :yellow
+              when :started, :start, :streamming
+                :cyan
+              else
+                :cyan
+              end
+    end
+
+    color = SEVERITY_COLOR[color] if Integer === color
+    color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
+    color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
+
+    str = str.to_s unless str.nil?
+    return str if Symbol === color
+    color_str = reset ? Term::ANSIColor.reset : ""
+    color_str << color
     if str.nil?
-      color
+      color_str
     else
-      color + str.to_s + self.color(0)
+      color_str + str.to_s + Term::ANSIColor.reset
     end
   end
 

data/lib/scout/log/progress/report.rb

@@ -195,15 +195,16 @@ module Log
     def done(io = STDERR)
       done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
       if @start
-        ellapsed = (Time.now - @start).to_i
+        ellapsed = (Time.now - @start)
       else
         ellapsed = 0
       end
-      ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
-      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
+      ellapsed_str = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed_str)
       @last_count = 0
       @last_time = @start
-      done_msg << " - " << thr_msg 
+      thr = ellapsed > 0 ? (@ticks / ellapsed).to_i.to_s : 0
+      done_msg << " - " << Log.color(:blue, thr) << " per second"
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 

data/lib/scout/log/progress/util.rb

@@ -28,6 +28,8 @@ module Log
     end
 
     def self.new_bar(max, options = {})
+      options, max = max, nil if Hash === max
+      max = options[:max] if max.nil?
       cleanup_bars
       BAR_MUTEX.synchronize do
         Log::LAST.replace "new_bar" if Log::LAST == "progress"

data/lib/scout/log/progress.rb

@@ -24,6 +24,8 @@ module Log
         IndiferentHash.process_options options, :depth, :desc, :file, :bytes, :frequency, :process, :callback,
         :depth => 0, :frequency => 2
 
+      max = nil if TrueClass === max
+
       @max = max
       @ticks = 0
       @frequency = frequency

data/lib/scout/log.rb

@@ -206,7 +206,11 @@ module Log
       line = line.sub('`',"'")
       color = :green if line =~ /workflow/
       color = :blue if line =~ /scout-/
-      Log.color color, line
+      if color
+        Log.color color, line
+      else
+        line
+      end
     end unless stack.nil?
   end
 

data/lib/scout/misc/digest.rb

@@ -41,9 +41,7 @@ module Misc
 
   def self.digest(obj)
     str = Misc.digest_str(obj)
-    hash = Digest::MD5.hexdigest(str)
-    Log.debug "Digest #{hash} - #{str}"
-    hash
+    Digest::MD5.hexdigest(str)
   end
 
   def self.file_md5(file)

data/lib/scout/misc/monitor.rb

@@ -20,4 +20,22 @@ module Misc
     end
     res
   end
+
+  def self.profile(options = {})
+    require 'ruby-prof'
+    profiler = RubyProf::Profile.new
+    profiler.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = profiler.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
 end

data/lib/scout/open/stream.rb

@@ -31,13 +31,16 @@ module Open
         Thread.current.report_on_exception = false
         consume_stream(io, false, into, into_close)
       end
+
       io.threads.push(consumer_thread) if io.respond_to?(:threads)
+      Thread.pass until consumer_thread["name"]
+
       consumer_thread
     else
       if into
-        Log.medium "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
+        Log.low "Consuming stream #{Log.fingerprint io} -> #{Log.fingerprint into}"
       else
-        Log.medium "Consuming stream #{Log.fingerprint io}"
+        Log.low "Consuming stream #{Log.fingerprint io}"
       end
 
       begin
@@ -53,7 +56,6 @@ module Open
         into_close = false unless into.respond_to? :close
         io.sync = true
 
-        Log.high "started consuming stream #{Log.fingerprint io}"
         begin
           while c = io.readpartial(BLOCK_SIZE)
             into << c if into
@@ -67,15 +69,14 @@ module Open
         into.close if into and into_close and not into.closed?
         block.call if block_given?
 
-        Log.high "Done consuming stream #{Log.fingerprint io} into #{into_path || into}"
         c
       rescue Aborted
-        Log.high "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
+        Log.low "Consume stream Aborted #{Log.fingerprint io} into #{into_path || into}"
         io.abort $! if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
         FileUtils.rm into_path if into_path and File.exist?(into_path)
       rescue Exception
-        Log.high "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
+        Log.low "Consume stream Exception reading #{Log.fingerprint io} into #{into_path || into} - #{$!.message}"
         exception = io.stream_exception || $!
         io.abort exception if io.respond_to? :abort
         into.close if into.respond_to?(:closed?) && ! into.closed?
@@ -145,12 +146,12 @@ module Open
 
           Open.notify_write(path) 
         rescue Aborted
-          Log.medium "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
+          Log.low "Aborted sensible_write -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
         rescue Exception
           exception = (AbortedStream === content and content.exception) ? content.exception : $!
-          Log.medium "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
+          Log.low "Exception in sensible_write: [#{Process.pid}] #{exception.message} -- #{ Log.color :blue, path }"
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise exception
@@ -219,16 +220,15 @@ module Open
 
       #parent_pid = Process.pid
       pid = Process.fork {
-        purge_pipes(sin)
-        sout.close
         begin
+          purge_pipes(sin)
+          sout.close
 
           yield sin
           sin.close if close and not sin.closed? 
 
         rescue Exception
           Log.exception $!
-          #Process.kill :INT, parent_pid
           Kernel.exit!(-1)
         end
         Kernel.exit! 0
@@ -242,18 +242,18 @@ module Open
       ConcurrentStream.setup sout, :pair => sin
 
       thread = Thread.new do 
-        Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
-        Thread.current.report_on_exception = false
         begin
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Pipe input #{Log.fingerprint sin} => #{Log.fingerprint sout}"
           
           yield sin
 
           sin.close if close and not sin.closed? and not sin.aborted?
         rescue Aborted
-          Log.medium "Aborted open_pipe: #{$!.message}"
+          Log.low "Aborted open_pipe: #{$!.message}"
           raise $!
         rescue Exception
-          Log.medium "Exception in open_pipe: #{$!.message}"
+          Log.low "Exception in open_pipe: #{$!.message}"
           begin
             sout.threads.delete(Thread.current)
             sout.pair = []
@@ -269,6 +269,7 @@ module Open
 
       sin.threads = [thread]
       sout.threads = [thread]
+      Thread.pass until thread["name"]
     end
 
     sout
@@ -287,8 +288,8 @@ module Open
 
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
-        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
         Thread.current.report_on_exception = false
+        Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
         begin
@@ -317,7 +318,7 @@ module Open
         out_pipes.each do |sout|
           sout.abort if sout.respond_to? :abort
         end
-        Log.medium "Tee aborting #{Log.fingerprint stream}"
+        Log.low "Tee aborting #{Log.fingerprint stream}"
         raise $!
       rescue Exception
         begin
@@ -332,7 +333,7 @@ module Open
           in_pipes.each do |sin|
             sin.close unless sin.closed?
           end
-          Log.medium "Tee exception #{Log.fingerprint stream}"
+          Log.low "Tee exception #{Log.fingerprint stream}"
         rescue
           Log.exception $!
         ensure
@@ -348,7 +349,7 @@ module Open
     out_pipes.each do |sout|
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :pair => stream
     end
-    splitter_thread.wakeup until splitter_thread["name"]
+    Thread.pass until splitter_thread["name"]
 
     main_pipe = out_pipes.first
     main_pipe.autojoin = true
@@ -370,4 +371,34 @@ module Open
   def self.tee_stream(stream)
     tee_stream_thread(stream)
   end
+
+  def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+
+  def self.read_stream(stream, size)
+    str = ""
+    while str.length < size
+      missing = size - str.length
+      more = stream.read(missing)
+      str << more
+    end
+    str
+  end
+
 end

data/lib/scout/semaphore.rb

@@ -0,0 +1,148 @@
+begin
+  require 'inline'
+  continue = true
+rescue Exception
+  Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
+  continue = false
+end
+
+if continue
+  module ScoutSemaphore
+    inline(:C) do |builder|
+      builder.prefix <<-EOF
+  #include <unistd.h>
+  #include <stdio.h>
+  #include <stdlib.h>
+  #include <semaphore.h>
+  #include <time.h>
+  #include <assert.h>
+  #include <errno.h>
+  #include <signal.h>
+  #include <fcntl.h>
+      EOF
+
+      builder.c_singleton <<-EOF
+  void create_semaphore(char* name, int value){
+    sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+  }
+      EOF
+      builder.c_singleton <<-EOF
+  void delete_semaphore(char* name){
+    sem_unlink(name);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  int wait_semaphore(char* name){
+    int ret;
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    ret = sem_wait(sem);
+    sem_close(sem);
+    return(ret);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  void post_semaphore(char* name){
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    sem_post(sem);
+    sem_close(sem);
+  }
+      EOF
+    end
+
+    SEM_MUTEX = Mutex.new
+    def self.synchronize(sem)
+      ret = ScoutSemaphore.wait_semaphore(sem)
+      raise SemaphoreInterrupted if ret == -1
+      begin
+        yield
+      ensure
+        ScoutSemaphore.post_semaphore(sem)
+      end
+    end
+
+    def self.with_semaphore(size, file = nil)
+      if file.nil?
+        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+      else
+        file = file.gsub('/', '_') if file
+      end
+
+      begin
+        Log.debug "Creating semaphore (#{ size }): #{file}"
+        ScoutSemaphore.create_semaphore(file, size)
+        yield file
+      ensure
+        Log.debug "Removing semaphore #{ file }"
+        ScoutSemaphore.delete_semaphore(file)
+      end
+    end
+
+    def self.fork_each_on_semaphore(elems, size, file = nil)
+
+      TSV.traverse elems, :cpus => size, :bar => "Fork each on semaphore: #{ Misc.fingerprint elems }", :into => Set.new do |elem|
+        elems.annotate elem if elems.respond_to? :annotate
+        begin
+          yield elem
+        rescue Interrupt
+          Log.warn "Process #{Process.pid} was aborted"
+        end
+        nil
+      end
+      nil
+    end
+
+    def self.thread_each_on_semaphore(elems, size)
+      mutex = Mutex.new
+      count = 0
+      cv = ConditionVariable.new
+      wait_mutex = Mutex.new
+
+      begin
+
+        threads = []
+        wait_mutex.synchronize do
+          threads = elems.collect do |elem| 
+            Thread.new(elem) do |elem|
+
+              continue = false
+              mutex.synchronize do
+                while not continue do
+                  if count < size 
+                    continue = true
+                    count += 1
+                  end
+                  mutex.sleep 1 unless continue
+                end
+              end
+
+              begin
+                yield elem
+              rescue Interrupt
+                Log.error "Thread was aborted while processing: #{Misc.fingerprint elem}"
+                raise $!
+              ensure
+                mutex.synchronize do
+                  count -= 1
+                  cv.signal if mutex.locked?
+                end
+              end
+            end
+          end
+        end
+
+        threads.each do |thread| 
+          thread.join 
+        end
+      rescue Exception
+        Log.exception $!
+        Log.info "Ensuring threads are dead: #{threads.length}"
+        threads.each do |thread| thread.kill end
+      end
+    end
+  end 
+end
+