scout-gear 5.2.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c39189bc9fe8816a3c0494c78966486f28383e8e15bf38ab33fe4dcde6739f3c
-  data.tar.gz: 5f2753e6810a68be44589f9392de1eff2f8592dd2704b05a0ccdf866f07f6a1b
+  metadata.gz: 919678993bd756d6a67712023120e562c01a69942faf8ad23eb088f29559b1fa
+  data.tar.gz: a233c808dc9af64381196bce97c74727c9064c9b1a4c2a129b364a5d776b01e1
 SHA512:
-  metadata.gz: 820d17306f468e7b206672b717489f78eb6fbfa37c843441202ce124c54cb7b84b89062a951c14a2db65ada47faf073dbc6a1b863294388ce66a8c1f7dd36e9f
-  data.tar.gz: c0fe23608355128c0e709db78c5598c5ece26e885eaa8788872878b7d55e29fc50bcb8215d3b4080f9e8d89456630e2739b087ee7d8b4b57498b299df3dc89d2
+  metadata.gz: 842323af61a5c66c7bf4433cbc0d23e6484e562c88601c64ac7a47ea2c1e217e6f7b590bd7d613d62614ebb2ab228246a2ac150d4d263f9425d9c1a2b4333e65
+  data.tar.gz: 2a4a7d00279d0cbc3153d7b116405fb5d02f8656249b4e0bdf417d01599749686242fb30db183b2baff856e92dcd5c83317161e73bb5bb1dbc861ce10d0086de

data/.vimproject

@@ -103,6 +103,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     usage.rb
     util.rb
    }
+   semaphore.rb
+   work_queue.rb
+   work_queue=work_queue{
+    socket.rb
+    worker.rb
+   }
   }
  }
  test=test {
@@ -224,16 +230,16 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       concurrency=concurrency{
        processes.rb
        threads.rb
-     processes=processes{
-      socket.rb
-      worker.rb
-     }
-    }
+       processes=processes{
+        socket.rb
+        worker.rb
+       }
+      }
 
-    procpath.rb
+      procpath.rb
 
-    migrate.rb
-   }
+      migrate.rb
+     }
 
    monitor.rb
 

data/VERSION

@@ -1 +1 @@
-5.2.0
+6.0.0

data/lib/scout/exceptions.rb

@@ -71,8 +71,20 @@ end
 
 class LockInterrupted < TryAgain; end
 
-#
-#class ClosedStream < StandardError; end
+class ClosedStream < StandardError; end
+
+class DoneProcessing < Exception
+  attr_accessor :pid
+  def initialize(pid = Process.pid)
+    @pid = pid
+  end
+
+  def message
+    "Done processing pid #{pid}"
+  end
+end
+
+
 #class OpenGzipError < StandardError; end
 #
 #

data/lib/scout/log/color.rb

@@ -2,6 +2,7 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
+require 'colorist'
 
 module Colorize
   def self.colors=(colors)
@@ -9,7 +10,13 @@ module Colorize
   end
 
   def self.colors
-    @colors ||= IndiferentHash.setup({green: "#00cd00" , red: "#cd0000" , yellow: "#ffd700" })
+    @colors ||= IndiferentHash.setup(Hash[<<-EOF.split("\n").collect{|l| l.split(" ")}])
+green #00cd00 
+red #cd0000 
+yellow #ffd700
+blue #0000cd
+path blue
+EOF
   end
 
   def self.diverging_colors=(colors)
@@ -33,7 +40,6 @@ module Colorize
     EOF
   end
 
-
   def self.from_name(color)
     return color if color =~ /^#?[0-9A-F]+$/i
     return colors[color.to_s] if colors.include?(color.to_s)
@@ -52,7 +58,7 @@ module Colorize
     when 'blue'
       colors["RoyalBlue"] 
     else
-      colors[color.to_s] || color
+      colors[color.to_s] || color.to_s
     end
   end
 
@@ -134,8 +140,21 @@ module Log
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
   SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
+  CONCEPT_COLORS = IndiferentHash.setup({
+    :title => magenta,
+    :path => blue,
+    :input => blue,
+    :value => green,
+    :integer => green,
+    :negative => red,
+    :float => green,
+    :waiting => yellow,
+    :started => blue,
+    :done => green,
+    :error => red,
+  })
   HIGHLIGHT = "\033[1m"
-
+  
   def self.uncolor(str)
     "" << Term::ANSIColor.uncolor(str)
   end
@@ -144,15 +163,20 @@ module Log
     reset
   end
 
-  def self.color(severity, str = nil, reset = false)
+  def self.color(color, str = nil, reset = false)
     return str.dup || "" if nocolor 
-    color = reset ? Term::ANSIColor.reset : ""
-    color << SEVERITY_COLOR[severity] if Integer === severity
-    color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
+
+    color = SEVERITY_COLOR[color] if Integer === color
+    color = CONCEPT_COLORS[color] if CONCEPT_COLORS.include?(color)
+    color = Term::ANSIColor.send(color) if Symbol === color and Term::ANSIColor.respond_to?(color)
+
+    return str if Symbol === color
+    color_str = reset ? Term::ANSIColor.reset : ""
+    color_str << color
     if str.nil?
-      color
+      color_str
     else
-      color + str.to_s + self.color(0)
+      color_str + str.to_s + Term::ANSIColor.reset
     end
   end
 

data/lib/scout/log/progress/report.rb

@@ -195,15 +195,16 @@ module Log
     def done(io = STDERR)
       done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
       if @start
-        ellapsed = (Time.now - @start).to_i
+        ellapsed = (Time.now - @start)
       else
         ellapsed = 0
       end
-      ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
-      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
+      ellapsed_str = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed_str)
       @last_count = 0
       @last_time = @start
-      done_msg << " - " << thr_msg 
+      thr = ellapsed > 0 ? (@ticks / ellapsed).to_i.to_s : 0
+      done_msg << " - " << Log.color(:blue, thr) << " per second"
       done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
 

data/lib/scout/misc/monitor.rb

@@ -20,4 +20,22 @@ module Misc
     end
     res
   end
+
+  def self.profile(options = {})
+    require 'ruby-prof'
+    profiler = RubyProf::Profile.new
+    profiler.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = profiler.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
 end

data/lib/scout/open/stream.rb

@@ -370,4 +370,34 @@ module Open
   def self.tee_stream(stream)
     tee_stream_thread(stream)
   end
+
+  def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+
+  def self.read_stream(stream, size)
+    str = ""
+    while str.length < size
+      missing = size - str.length
+      more = stream.read(missing)
+      str << more
+    end
+    str
+  end
+
 end

data/lib/scout/semaphore.rb

@@ -0,0 +1,148 @@
+begin
+  require 'inline'
+  continue = true
+rescue Exception
+  Log.warn "The RubyInline gem could not be loaded: semaphore synchronization will not work"
+  continue = false
+end
+
+if continue
+  module ScoutSemaphore
+    inline(:C) do |builder|
+      builder.prefix <<-EOF
+  #include <unistd.h>
+  #include <stdio.h>
+  #include <stdlib.h>
+  #include <semaphore.h>
+  #include <time.h>
+  #include <assert.h>
+  #include <errno.h>
+  #include <signal.h>
+  #include <fcntl.h>
+      EOF
+
+      builder.c_singleton <<-EOF
+  void create_semaphore(char* name, int value){
+    sem_open(name, O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO, value);
+  }
+      EOF
+      builder.c_singleton <<-EOF
+  void delete_semaphore(char* name){
+    sem_unlink(name);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  int wait_semaphore(char* name){
+    int ret;
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    ret = sem_wait(sem);
+    sem_close(sem);
+    return(ret);
+  }
+      EOF
+
+      builder.c_singleton <<-EOF
+  void post_semaphore(char* name){
+    sem_t* sem;
+    sem = sem_open(name, 0);
+    sem_post(sem);
+    sem_close(sem);
+  }
+      EOF
+    end
+
+    SEM_MUTEX = Mutex.new
+    def self.synchronize(sem)
+      ret = ScoutSemaphore.wait_semaphore(sem)
+      raise SemaphoreInterrupted if ret == -1
+      begin
+        yield
+      ensure
+        ScoutSemaphore.post_semaphore(sem)
+      end
+    end
+
+    def self.with_semaphore(size, file = nil)
+      if file.nil?
+        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+      else
+        file = file.gsub('/', '_') if file
+      end
+
+      begin
+        Log.debug "Creating semaphore (#{ size }): #{file}"
+        ScoutSemaphore.create_semaphore(file, size)
+        yield file
+      ensure
+        Log.debug "Removing semaphore #{ file }"
+        ScoutSemaphore.delete_semaphore(file)
+      end
+    end
+
+    def self.fork_each_on_semaphore(elems, size, file = nil)
+
+      TSV.traverse elems, :cpus => size, :bar => "Fork each on semaphore: #{ Misc.fingerprint elems }", :into => Set.new do |elem|
+        elems.annotate elem if elems.respond_to? :annotate
+        begin
+          yield elem
+        rescue Interrupt
+          Log.warn "Process #{Process.pid} was aborted"
+        end
+        nil
+      end
+      nil
+    end
+
+    def self.thread_each_on_semaphore(elems, size)
+      mutex = Mutex.new
+      count = 0
+      cv = ConditionVariable.new
+      wait_mutex = Mutex.new
+
+      begin
+
+        threads = []
+        wait_mutex.synchronize do
+          threads = elems.collect do |elem| 
+            Thread.new(elem) do |elem|
+
+              continue = false
+              mutex.synchronize do
+                while not continue do
+                  if count < size 
+                    continue = true
+                    count += 1
+                  end
+                  mutex.sleep 1 unless continue
+                end
+              end
+
+              begin
+                yield elem
+              rescue Interrupt
+                Log.error "Thread was aborted while processing: #{Misc.fingerprint elem}"
+                raise $!
+              ensure
+                mutex.synchronize do
+                  count -= 1
+                  cv.signal if mutex.locked?
+                end
+              end
+            end
+          end
+        end
+
+        threads.each do |thread| 
+          thread.join 
+        end
+      rescue Exception
+        Log.exception $!
+        Log.info "Ensuring threads are dead: #{threads.length}"
+        threads.each do |thread| thread.kill end
+      end
+    end
+  end 
+end
+

data/lib/scout/work_queue/socket.rb

@@ -0,0 +1,119 @@
+require 'scout/open'
+require 'scout/semaphore'
+require 'scout/exceptions'
+class WorkQueue
+  class Socket
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
+    def initialize(serializer = nil)
+      @sread, @swrite = Open.pipe
+
+      @serializer = serializer || Marshal
+
+      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @write_sem = @key + '.in'
+      @read_sem = @key + '.out'
+      Log.debug "Creating socket semaphores: #{@key}"
+      ScoutSemaphore.create_semaphore(@write_sem,1)
+      ScoutSemaphore.create_semaphore(@read_sem,1)
+    end
+
+    def clean
+      @cleaned = true
+      @sread.close unless @sread.closed?
+      @swrite.close unless @swrite.closed?
+      Log.low "Destroying socket semaphores: #{[@key] * ", "}"
+      ScoutSemaphore.delete_semaphore(@write_sem)
+      ScoutSemaphore.delete_semaphore(@read_sem)
+    end
+
+
+    def dump(obj)
+      stream = @swrite
+      obj.concurrent_stream = nil if obj.respond_to?(:concurrent_stream)
+      case obj
+      when Integer
+        size_head = [obj,"I"].pack 'La'
+        str = size_head
+      when nil
+        size_head = [0,"N"].pack 'La'
+        str = size_head
+      when String
+        payload = obj
+        size_head = [payload.bytesize,"C"].pack 'La'
+        str = size_head << payload
+      else
+        payload = @serializer.dump(obj)
+        size_head = [payload.bytesize,"S"].pack 'La'
+        str = size_head << payload
+      end
+
+      write_length = str.length
+      wrote = stream.write(str) 
+      while wrote < write_length
+        wrote += stream.write(str[wrote..-1]) 
+      end
+    end
+
+    def load
+      stream = @sread
+      size_head = Open.read_stream stream, 5
+
+      size, type = size_head.unpack('La')
+
+      return nil if type == "N"
+      return size.to_i if type == "I"
+      begin
+        payload = Open.read_stream stream, size
+        case type
+        when "S"
+          begin
+            @serializer.load(payload)
+          rescue Exception
+            Log.exception $!
+            raise $!
+          end
+        when "C"
+          payload
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+
+    def closed_read?
+      @sread.closed?
+    end
+
+    def closed_write?
+      @swrite.closed?
+    end
+
+    def close_write
+      self.dump ClosedStream.new
+      @swrite.close unless closed_write?
+    end
+
+    def close_read
+      @sread.close unless closed_read?
+    end
+
+    #{{{ ACCESSOR
+    def push(obj)
+      ScoutSemaphore.synchronize(@write_sem) do
+        self.dump(obj)
+      end
+    end
+
+    def pop
+      ScoutSemaphore.synchronize(@read_sem) do
+        res = self.load
+        raise res if ClosedStream === res
+        res
+      end
+    end
+
+    alias write push
+
+    alias read pop
+  end
+end

data/lib/scout/work_queue/worker.rb

@@ -0,0 +1,54 @@
+class WorkQueue
+  class Worker
+    attr_accessor :pid, :ignore_ouput
+    def initialize
+    end
+
+    def run
+      @pid = Process.fork do
+        yield
+      end
+    end
+
+    def process(input, output, &block)
+      run do
+        begin
+          while obj = input.read
+            if DoneProcessing === obj
+              output.write DoneProcessing.new
+              raise obj 
+            end
+            res = block.call obj
+            output.write res unless ignore_ouput || res == :ignore 
+          end
+        rescue DoneProcessing
+          Log.log "Worker #{Process.pid} done"
+        rescue Exception
+          Log.exception $!
+          exit -1
+        end
+      end
+    end
+
+    def join
+      Log.log "Joining worker #{@pid}"
+      Process.waitpid @pid
+    end
+
+    def exit(status)
+      Log.log "Worker #{@pid} exited with status #{Log.color(:green, status)}"
+    end
+
+    def self.join(workers)
+      workers = [workers] unless Array === workers
+      begin
+        while pid = Process.wait 
+          status = $?
+          worker = workers.select{|w| w.pid == pid }.first
+          worker.exit status.exitstatus if worker
+        end
+      rescue Errno::ECHILD
+      end
+    end
+  end
+end

data/lib/scout/work_queue.rb

@@ -0,0 +1,86 @@
+require_relative 'work_queue/socket'
+require_relative 'work_queue/worker'
+
+class WorkQueue
+  attr_accessor :workers, :worker_proc, :callback
+
+  def initialize(workers = 0, &block)
+    @input = WorkQueue::Socket.new
+    @output = WorkQueue::Socket.new
+    @workers = workers.times.collect{ Worker.new }
+    @worker_proc = block
+    @worker_mutex = Mutex.new
+    @removed_workers = []
+  end
+
+  def add_worker(&block)
+    worker = Worker.new
+    @worker_mutex.synchronize do
+      @workers.push(worker)
+      if block_given?
+        worker.process @input, @output, &block
+      else
+        worker.process @input, @output, &@worker_proc
+      end
+    end
+    worker
+  end
+
+  def ignore_ouput
+    @workers.each{|w| w.ignore_ouput = true }
+  end
+
+  def remove_one_worker
+    @input.write DoneProcessing.new
+  end
+
+  def remove_worker(pid)
+    worker = @worker_mutex.synchronize do
+      Log.debug "Remove #{pid}"
+      @removed_workers.concat(@workers.delete_if{|w| w.pid == pid })
+    end
+  end
+
+  def process(&callback)
+    @workers.each do |w| 
+      w.process @input, @output, &@worker_proc
+    end
+    @reader = Thread.new do
+      begin
+        while true
+          obj = @output.read
+          if DoneProcessing === obj
+            remove_worker obj.pid if obj.pid
+          else
+            callback.call obj if callback
+          end
+        end
+      rescue Aborted
+      end
+    end if @output
+  end
+
+  def write(obj)
+    @input.write obj
+  end
+
+  def close
+    while @worker_mutex.synchronize{ @workers.length } > 0
+      begin
+        @input.write DoneProcessing.new
+        pid = Process.wait
+        status = $?
+        worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
+        worker.exit $?.exitstatus if worker
+      rescue Errno::ECHILD
+        Thread.pass until @workers.length == 0
+        break
+      end
+    end
+    @reader.raise Aborted if @reader
+  end
+
+  def join
+    @reader.join if @reader
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -55,9 +55,9 @@ class Step
   
   def report_status(status, message = nil)
     if message.nil?
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
+      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path)
     else
-      Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
+      Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path) + " " + message
     end
   end