rbbt-util 5.10.1 → 5.10.2

data/lib/rbbt/tsv/parser.rb

@@ -16,6 +16,7 @@ module TSV
 
       # Get line
 
+      Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
       line = stream.gets
       raise "Empty content" if line.nil?
       line = Misc.fixutf8 line
@@ -40,6 +41,7 @@ module TSV
         @key_field = @fields.shift
         @key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
 
+        Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
         line = @header_hash != "" ?  Misc.fixutf8(stream.gets) : nil
       end
 
@@ -348,7 +350,8 @@ module TSV
 
       options = header_options.merge options
 
-      @type = Misc.process_options(options, :type) || :double
+      @type ||= Misc.process_options(options, :type) || :double
+      @type ||= :double
 
       @filename = Misc.process_options(options, :filename) 
       @filename ||= stream.filename if stream.respond_to? :filename
@@ -497,7 +500,10 @@ module TSV
             
             yield key, values
 
+            Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
+
             line = stream.gets
+
             line_num += 1
             raise END_PARSING if head and line_num > head.to_i
           rescue SKIP_LINE
@@ -514,8 +520,6 @@ module TSV
             break
           end
         end
-      #ensure
-      #  stream.close unless stream.closed?
       end
 
       self

data/lib/rbbt/tsv/util.rb

@@ -98,7 +98,10 @@ module TSV
     when Path
       file.open(open_options)
     when IO, StringIO
-      file.rewind if file.respond_to?(:rewind) and file.eof?
+      begin
+        file.rewind if file.respond_to?(:rewind) and file.eof?
+      rescue
+      end
       file
     when String
       raise "Could not open file given by String: #{Misc.fingerprint file}" unless Open.remote?(file) or File.exists? file

data/lib/rbbt/util/cmd.rb

@@ -124,7 +124,7 @@ module CMD
 
     in_content = StringIO.new in_content if String === in_content
 
-    sout, serr, sin = IO.pipe, IO.pipe, IO.pipe
+    sout, serr, sin = Misc.pipe, Misc.pipe, Misc.pipe
 
     pid = fork {
       begin

data/lib/rbbt/util/concurrency/processes.rb

@@ -1,14 +1,14 @@
 require 'rbbt/util/concurrency/processes/worker'
 require 'rbbt/util/concurrency/processes/socket'
 
-
 class RbbtProcessQueue
   #{{{ RbbtProcessQueue
 
-  attr_accessor :num_processes, :processes, :queue, :process_monitor
-  def initialize(num_processes)
+  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup
+  def initialize(num_processes, cleanup = nil)
     @num_processes = num_processes
     @processes = []
+    @cleanup = cleanup
     @queue = RbbtProcessSocket.new
   end
 
@@ -24,13 +24,16 @@ class RbbtProcessQueue
           loop do
             p = @callback_queue.pop
             raise p if Exception === p
+            raise p.first if Array === p and Exception === p.first
             @callback.call p
           end
         rescue ClosedStream
         rescue Exception
-          Log.debug $!
+          Log.exception $!
+          sleep 1
           parent.raise $!
-          Thread.exit
+        ensure
+          @callback_queue.sread.close unless @callback_queue.sread.closed?
         end
       end
     else
@@ -40,47 +43,51 @@ class RbbtProcessQueue
 
   def init(&block)
     num_processes.times do |i|
-      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, &block)
+      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, &block)
     end
-    @queue.sread.close
-    @callback_queue.swrite.close if @callback_queue
+    @queue.close_read
 
     @process_monitor = Thread.new(Thread.current) do |parent|
       begin
-        while @processes.any? do
-          pid = Process.wait -1, Process::WNOHANG
-          if pid
-            next unless @processes.collect{|p| p.pid }.include? pid
-            @processes.delete_if{|p| p.pid == pid}
-            raise "Process #{pid} failed" unless $?.success?
-          else
-            sleep 1
+        while @processes.any?
+          @processes[0].join 
+          @processes.shift
+        end
+      rescue Exception
+        @processes.each do |p|
+          begin
+            Process.kill :INT, p
+          rescue
           end
         end
-      rescue
+        Log.exception $!
         parent.raise $!
-      ensure
-        Thread.exit
       end
     end
   end
 
   def close_callback
-    @callback_thread.join if @callback_thread and @callback_thread.alive?
+    @callback_queue.push ClosedStream.new if  @callback_thread.alive?
+    @callback_queue.swrite.close 
+    @callback_thread.join 
   end
 
   def join
-    @queue.push ClosedStream.new
-    @queue.swrite.close
+    @processes.length.times do 
+      @queue.push ClosedStream.new
+    end
     begin
       @process_monitor.join
-    ensure
       close_callback if @callback
+    rescue Exception
+      Log.exception $!
+    ensure
+      @queue.swrite.close
     end
   end
 
   def clean
-    @processes.each{|p| p.abort }.clear
+    @processes.each{|p| p.abort }
     @callback_thread.raise Aborted if @callback_thread and @callback_thread.alive?
   end
 

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -7,7 +7,7 @@ class RbbtProcessQueue
 
     attr_accessor :sread, :swrite, :write_sem, :read_sem
     def initialize
-      @sread, @swrite = IO.pipe
+      @sread, @swrite = Misc.pipe
 
       key = rand(100000).to_s;
       @write_sem = key + '.in'
@@ -62,25 +62,33 @@ class RbbtProcessQueue
       end
     end
 
+    def closed_read?
+      @sread.closed?
+    end
+
+    def closed_write?
+      @swrite.closed?
+    end
+
+    def close_write
+      @swrite.close
+    end
+
+    def close_read
+      @sread.close 
+    end
     #{{{ ACCESSOR
+  
     
     def push(obj)
-      begin
-        RbbtSemaphore.synchronize(@write_sem) do
-          self.dump(obj, @swrite)
-        end
-      rescue
-        return ClosedStream.new
+      RbbtSemaphore.synchronize(@write_sem) do
+        self.dump(obj, @swrite)
       end
     end
 
     def pop
-      begin
-        RbbtSemaphore.synchronize(@read_sem) do
-          self.load(@sread)
-        end
-      rescue IOError, ClosedStream
-        return ClosedStream.new
+      RbbtSemaphore.synchronize(@read_sem) do
+        self.load(@sread)
       end
     end
   end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -1,14 +1,21 @@
 require 'rbbt/util/concurrency/processes/socket'
 class RbbtProcessQueue
   class RbbtProcessQueueWorker
-    attr_accessor :pid, :queue, :callback_queue, :block
-    def initialize(queue, callback_queue = nil, &block)
-      @queue, @callback_queue, @block = queue, callback_queue, block
+    attr_reader :pid, :queue, :callback_queue, :cleanup, :block
+    def initialize(queue, callback_queue = nil, cleanup = nil, &block)
+      @queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
 
       @pid = Process.fork do
         begin
-          @queue.swrite.close
-          @callback_queue.sread.close if @callback_queue
+          @cleanup.call if @cleanup
+          @queue.close_write 
+
+          if @callback_queue
+            Misc.purge_pipes(@callback_queue.swrite) 
+            @callback_queue.close_read 
+          else
+            Misc.purge_pipes
+          end
 
           Signal.trap(:INT){ raise Aborted; }
           loop do
@@ -19,26 +26,29 @@ class RbbtProcessQueue
             @callback_queue.push res if @callback_queue
           end
 
-          exit 0
         rescue ClosedStream
-          exit 0
         rescue Aborted
-          exit -1
-        rescue Exception
           Log.exception $!
+        rescue Exception
           @callback_queue.push($!) if @callback_queue
-          exit -1
+        ensure
+          @callback_queue.close_write if @callback_queue 
         end
-
       end
     end
 
     def join
-      Process.waitpid @pid
+      begin
+        joined_pid = Process.waitpid @pid
+      rescue
+      end
     end
 
     def abort
-      Process.kill :INT, @pid
+      begin
+        Process.kill :INT, @pid
+      rescue
+      end
     end
 
     def done?

data/lib/rbbt/util/log.rb

@@ -17,8 +17,7 @@ module Log
   end
   self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
   require "highline/system_extensions.rb"
-  self.tty_size = HighLine::SystemExtensions.terminal_size.first
-
+  self.tty_size = HighLine::SystemExtensions.terminal_size.first 
 
   def self.with_severity(level)
     orig = Log.severity
@@ -60,7 +59,7 @@ module Log
   end
 
   def self.clear_line(out = STDOUT)
-    out.puts Log.return_line << " " * Log.tty_size << Log.return_line unless nocolor
+    out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
   end
 
   def self.highlight(str = nil)
@@ -74,7 +73,7 @@ module Log
   end
 
   def self.log(message = nil, severity = MEDIUM, &block)
-    return if severity < self.severity
+    return if severity < self.severity 
     message ||= block.call if block_given?
     return if message.nil?
 
@@ -195,6 +194,21 @@ def iii(message, file = $stdout)
   Log.info{""}
 end
 
+def www(message, file = $stdout)
+  stack = caller
+  Log.warn{"#{Log.color :cyan, "INFO:"} " << stack.first}
+  Log.warn{""}
+  Log.warn{"=> " << message.inspect}
+  Log.warn{""}
+end
+
+def eee(message, file = $stdout)
+  stack = caller
+  Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}
+  Log.error{""}
+  Log.error{"=> " << message.inspect}
+  Log.error{""}
+end
 
 if __FILE__ == $0
   Log.severity = 0

data/lib/rbbt/util/misc.rb

@@ -28,10 +28,201 @@ module LaterString
   end
 end
 
+module ConcurrentStream
+  attr_accessor :threads, :pids, :callback, :filename
+
+  def consume
+    Thread.pass while IO.select([self], nil, nil).nil? #if IO === self
+    while block = self.read(2048)
+      Thread.pass while IO.select([self], nil, nil).nil? # if IO === self
+    end
+  end
+
+  def join
+    filename = self.respond_to?(:filename)? self.filename : :none
+    if @callback
+      @callback.call
+    end
+    @threads.each{|t| t.join } if @threads
+    
+    @pids.each do |pid| 
+      begin
+        Process.waitpid(pid, Process::WUNTRACED)
+        raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
+      rescue Errno::ECHILD
+      end
+    end if @pids
+  end
+
+  def self.setup(stream, options = {}, &block)
+    threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+
+    callback = block if block_given?
+    if stream.callback and callback
+      old_callback = stream.callback
+      stream.callback = Proc.new do
+        old_callback.call
+        callback.call
+      end
+    else
+      stream.callback = callback
+    end
+
+    stream.filename = filename unless filename.nil?
+
+    stream
+  end
+end
+
+
 Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
 module Misc
 
 
+  PIPE_MUTEX = Mutex.new
+
+  OPEN_PIPE_IN = []
+  def self.pipe
+    OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
+    PIPE_MUTEX.synchronize do
+      sout, sin = IO.pipe
+      OPEN_PIPE_IN << sin
+
+      [sout, sin]
+    end
+  end
+  
+  def self.release_pipes(*pipes)
+    PIPE_MUTEX.synchronize do
+      pipes.flatten.each do |pipe|
+        pipe.close unless pipe.closed?
+      end
+    end
+  end
+
+
+  def self.purge_pipes(*save)
+    PIPE_MUTEX.synchronize do
+      OPEN_PIPE_IN.each do |pipe|
+        next if save.include? pipe
+        pipe.close unless pipe.closed?
+      end
+    end
+  end
+
+  def self.open_pipe(do_fork = false, other_stream = nil)
+    raise "No block given" unless block_given?
+    sout, sin = Misc.pipe
+    if do_fork
+      parent_pid = Process.pid
+      pid = Process.fork {
+        purge_pipes(sin)
+        sout.close
+        begin
+          yield sin
+        rescue
+          Log.exception $!
+          Process.kill :INT, parent_pid
+          Kernel.exit! -1
+        ensure
+          Misc.release_pipes(sin)
+        end
+        Kernel.exit! 0
+      }
+      Misc.release_pipes(sin)
+    else
+      thread = Thread.new(Thread.current) do |parent|
+        begin
+          yield sin
+        rescue
+          Log.exception $!
+          parent.raise $!
+        ensure
+          Misc.release_pipes(sin)
+        end
+      end
+    end
+    sout
+  end
+
+  def self.tee_stream_fork(stream)
+    stream_out1, stream_in1 = Misc.pipe
+    stream_out2, stream_in2 = Misc.pipe
+
+    splitter_pid = Process.fork do
+      Misc.purge_pipes(stream_in1, stream_in2)
+      stream_out1.close
+      stream_out2.close
+      begin
+        filename = stream.respond_to?(:filename)? stream.filename : nil
+        while block = stream.read(2048)
+          begin stream_in1.write block; rescue Exception;  Log.exception $! end 
+          begin stream_in2.write block; rescue Exception;  Log.exception $! end 
+        end
+      rescue IOError
+        Log.exception $!
+      rescue Exception
+        Log.exception $!
+      ensure
+        if stream.respond_to? :join
+          stream.join
+        end
+        Misc.release_pipes(stream_in1)
+        Misc.release_pipes(stream_in2)
+      end
+    end
+    stream.close
+    stream_in1.close
+    stream_in2.close
+    #stream.join if stream.respond_to? :join
+
+    ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
+    ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
+
+    [stream_out1, stream_out2]
+  end
+
+  def self.tee_stream_thread(stream)
+    stream_out1, stream_in1 = Misc.pipe
+    stream_out2, stream_in2 = Misc.pipe
+
+    splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
+      begin
+        filename = stream.respond_to?(:filename)? stream.filename : nil
+        while block = stream.read(2048)
+          begin stream_in1.write block; rescue Exception;  Log.exception $! end 
+          begin stream_in2.write block; rescue Exception;  Log.exception $! end 
+        end
+      rescue IOError
+        Log.exception $!
+      rescue Exception
+        Log.exception $!
+        parent.raise $!
+      ensure
+        if stream.respond_to? :join
+          stream.join
+        end
+        Misc.release_pipes(stream_in1)
+        Misc.release_pipes(stream_in2)
+      end
+    end
+
+    ConcurrentStream.setup stream_out1, :threads => splitter_thread
+    ConcurrentStream.setup stream_out2, :threads => splitter_thread
+
+    [stream_out1, stream_out2]
+  end
+
+  class << self
+    alias tee_stream tee_stream_fork 
+  end
+
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -89,8 +280,10 @@ module Misc
 
   def self.read_stream(stream, size)
     str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
     while not str = stream.read(size)
       IO.select([stream],nil,nil,1) 
+      Thread.pass
       raise ClosedStream if stream.eof?
     end
 
@@ -1114,7 +1307,6 @@ end
         res = yield file, *args
       end
     rescue Interrupt
-      Log.error "Process #{Process.pid} interrupted while in lock: #{ lock_path }"
       raise $!
     end
 
@@ -1219,9 +1411,12 @@ end
             File.open(tmp_path, 'w', &block)
           when String === content
             File.open(tmp_path, 'w') do |f| f.write content end
-          when (IO === content or StringIO === content)
+          when (IO === content or StringIO === content or File === content)
+            #Thread.pass while IO.select([content], nil, nil, 1) if IO === content
             File.open(tmp_path, 'w') do |f|  
-              while l = content.gets; f.write l; end  
+              while block = content.read(2048); 
+                f.write block
+              end  
             end
           else
             File.open(tmp_path, 'w') do |f|  end
@@ -1512,21 +1707,6 @@ end
     chunks
   end
 
-  def self.open_pipe
-    sout, sin = IO.pipe
-    raise "No block given" unless block_given?
-    Thread.new{
-      begin
-        yield sin
-      rescue
-        Log.exception $!
-        raise $!
-      ensure
-        sin.close
-      end
-    }
-    sout
-  end
 
   def self.append_zipped(current, new)
     current.each do |v|