rbbt-util 5.20.2 → 5.20.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1edc4e03bed4c0f6255d0704137afec41d55018
-  data.tar.gz: b1460408f7d30542199e6420b3c498905b3abcf9
+  metadata.gz: c01ec7b654f4f713aaf4c9cf3b518aec087009de
+  data.tar.gz: 2721569f2e2a9445c97415eb5093150a6ca1850a
 SHA512:
-  metadata.gz: c4ec7f128cce7e7bd0e6e8791e1cea6b4c6b9903d6986baf9b1b6192ba588d0f5eeb271c276a23e0d07f9adaf53fbeafb1403f5e5c1ced19532957a95f59758d
-  data.tar.gz: 6406302f55c94fc5bbdd6809dd27a4d7b8cda95831623dde05f8bd0bce43cfe26b667a64e70e66af61b97ec4e10c9e110f2a7865ba601ad8f8dd0a293b0dbc7f
+  metadata.gz: c152113e19eaea2dac605abdff6e60132137dd1eb09f797befc230d2d4eb6422ce5c9c0defaaab6d6f684e772c0d65776efc0836ffacbf680b9881d1e4a13d41
+  data.tar.gz: 68a251d79736a2c0d54120f50071015c96a51270886f83ea951ca460fc1c0ae48af8f07ad3a90d1ed93a81af0acbfb7d3de7b57907ebaca741610e06e3de4740

data/lib/rbbt/association/index.rb

@@ -59,18 +59,16 @@ module Association
               list.replace [list.first] * size if list.length == 1
             end if recycle and size > 1
 
-
             rest = Misc.zip_fields rest
 
-
             annotations = (Array === rest.first and rest.first.length > 1) ?
               targets.zip(rest) :
               targets.zip(rest * targets.length) 
 
+            source = source.gsub('~','-..-')
             annotations.each do |target, info|
               next if target.nil? or target.empty?
-              source.gsub!('~','-..-')
-              target.gsub!('~','-..-')
+              target = target.gsub('~','-..-')
               key = [source, target] * "~"
 
               if data[key].nil? or info.nil?
@@ -160,6 +158,7 @@ module Association
 
                      new.undirected = undirected
 
+                     Log.tsv new
                      new
                    rescue Exception
                      Log.error "Deleting after error reversing database: #{ reverse_filename }"

data/lib/rbbt/tsv/accessor.rb

@@ -594,7 +594,7 @@ module TSV
     io = dumper_stream(keys, no_options)
 
     str = ''
-    while block = io.read(2048)
+    while block = io.read(Misc::BLOCK_SIZE)
       str << block
     end
 

data/lib/rbbt/tsv/manipulate.rb

@@ -179,7 +179,6 @@ module TSV
 
     each do |key, value|
       progress_monitor.tick if progress_monitor
-      next if value.nil?
 
       keys, value = traverser.process(key, value)
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -20,7 +20,13 @@ module TSV
       case obj
       when (defined? Step and Step)
         if obj.done?
-          CMD.cmd("wc -l '#{obj.path.find}'").read.to_i
+          path = obj.path
+          path = path.find if path.respond_to? :find
+          if File.exists? path
+            CMD.cmd("wc -l '#{path}'").read.to_i 
+          else
+            nil
+          end
         else
           nil
         end
@@ -349,7 +355,7 @@ module TSV
       respawn = true if ENV["RBBT_RESPAWN"] and ENV["RBBT_RESPAWN"] == "true"
 
       Log.low "Traversing in #{ num } cpus: #{respawn ? "respawn" : "no respawn"}"
-      q = RbbtProcessQueue.new num, cleanup, join, respawn
+      q = RbbtProcessQueue.new num, cleanup, join, respawn, !!bar
       callback = Proc.new{ bar.tick } if callback.nil? and bar
       q.callback &callback
       q.init &block

data/lib/rbbt/util/cmd.rb

@@ -175,7 +175,7 @@ module CMD
         begin
           loop do
             break if in_content.closed?
-            block = in_content.read 1024
+            block = in_content.read Misc::BLOCK_SIZE
             break if block.nil? or block.empty?
             sin.write block
           end

data/lib/rbbt/util/concurrency/processes.rb

@@ -3,13 +3,14 @@ require 'rbbt/util/concurrency/processes/socket'
 
 class RbbtProcessQueue
 
-  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join, :reswpan
-  def initialize(num_processes, cleanup = nil, join = nil, reswpan = nil)
+  attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join, :reswpan, :offset
+  def initialize(num_processes, cleanup = nil, join = nil, reswpan = nil, offset = false)
     @num_processes = num_processes
     @processes = []
     @cleanup = cleanup
     @join = join
     @respawn = reswpan
+    @offset = offset
     @queue = RbbtProcessSocket.new
   end
 
@@ -59,7 +60,7 @@ class RbbtProcessQueue
 
   def init(&block)
     num_processes.times do |i|
-      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, &block)
+      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &block)
     end
     @queue.close_read
 

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -120,11 +120,12 @@ class RbbtProcessQueue
       end
     end
 
-    def initialize(queue, callback_queue = nil, cleanup = nil, respawn = false, &block)
-      @queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
+    def initialize(queue, callback_queue = nil, cleanup = nil, respawn = false, offset = false, &block)
+      @queue, @callback_queue, @cleanup, @block, @offset = queue, callback_queue, cleanup, block, offset
 
       @pid = Process.fork do
         Misc.pre_fork
+        Log::ProgressBar.add_offset if @offset
 
         @cleanup.call if @cleanup
         @queue.close_write 
@@ -141,6 +142,7 @@ class RbbtProcessQueue
         else
           run
         end
+        Log::ProgressBar.remove_offset if @offset
       end
     end
 

data/lib/rbbt/util/log/progress/util.rb

@@ -5,11 +5,30 @@ module Log
     REMOVE = []
     SILENCED = []
 
+    def self.add_offset
+      @@offset = offset + 1
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
+    def self.remove_offset
+      @@offset = offset - 1
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
+
+    def self.offset
+      @@offset ||= 0
+      @@offset = 0 if @@offset < 0
+      @@offset
+    end
+
     def self.new_bar(max, options = {})
       cleanup_bars
       BAR_MUTEX.synchronize do
         #Log::LAST.replace "new_bar" if Log::LAST == "progress"
-        options = Misc.add_defaults options, :depth => BARS.length
+        options = Misc.add_defaults options, :depth => BARS.length + Log::ProgressBar.offset
         BARS << (bar = ProgressBar.new(max, options))
         bar
       end

data/lib/rbbt/util/misc/bgzf.rb

@@ -175,7 +175,7 @@ module Bgzf
     str = nil
     current = @data_offset
     while true
-      block = read(1024)
+      block = read(Misc::BLOCK_SIZE)
       break if block.empty?
       str = "" if str.nil?
       if i = block.index("\n")

data/lib/rbbt/util/misc/inspect.rb

@@ -65,7 +65,7 @@ module Misc
       else
         new = "{"
         obj.each do |k,v|
-          new << k.to_s << '=>' << fingerprint(v) << ' '
+          new << fingerprint(k) << '=>' << fingerprint(v) << ' '
         end
         if new.length > 1
            new[-1] =  "}"
@@ -249,8 +249,8 @@ module Misc
     str
   end
 
-
-  def self.obj2md5(obj)
+  
+  def self.obj2digest(obj)
     str = obj2str(obj)
 
     if str.empty?
@@ -259,4 +259,8 @@ module Misc
       digest(str)
     end
   end
+
+  def self.obj2md5(obj)
+    obj2digest(obj)
+  end
 end

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -22,7 +22,7 @@ module RbbtMutiplartPayload
   EOL = "\r\n"
 
   def self.mutex
-    @mutex ||= Mutex.new
+    @@mutex ||= Mutex.new
   end
 
   def self.input_header(name, filename = nil)
@@ -47,17 +47,19 @@ module RbbtMutiplartPayload
 
   def self.add_stream(io, name, content, filename = nil)
     header = input_header(name, filename)
-    io.write "--" + BOUNDARY + EOL + header + EOL  + EOL
+    io.write "--" + BOUNDARY + EOL + header + EOL
 
-    while line = content.gets
-      io.puts line
+    begin
+      while c = content.readpartial(Misc::BLOCK_SIZE)
+        io.write c
+      end
+    rescue EOFError
     end
     content.close
   end
 
   def self.close_stream(io)
     io.write "--" + BOUNDARY + "--" + EOL + EOL
-    io.write EOL
     io.close
   end
 
@@ -121,8 +123,7 @@ module RbbtMutiplartPayload
       req.body = sout.read
     end
 
-    Misc.open_pipe(true) do |sin|
-      sleep rand(10).to_f / 5
+    Misc.open_pipe do |sin|
       Net::HTTP.start(uri.hostname, uri.port) do |http|
         http.request(req) do |res|
           url_path = res["RBBT-STREAMING-JOB-URL"]
@@ -139,7 +140,6 @@ module RbbtMutiplartPayload
             sin.puts "BULK" if report_type
             sin.write res.body
           end
-          sin.close
         end
       end
     end

data/lib/rbbt/util/misc/pipes.rb

@@ -16,6 +16,8 @@ module Misc
     end
   end
 
+  BLOCK_SIZE=1024 * 8
+
   PIPE_MUTEX = Mutex.new
 
   OPEN_PIPE_IN = []
@@ -87,70 +89,136 @@ module Misc
     sout
   end
 
-  def self.tee_stream_thread(stream)
-    stream_out1, stream_in1 = Misc.pipe
-    stream_out2, stream_in2 = Misc.pipe
+  #def self.tee_stream_thread(stream)
+  #  stream_out1, stream_in1 = Misc.pipe
+  #  stream_out2, stream_in2 = Misc.pipe
+
+  #  splitter_thread = Thread.new(Thread.current) do |parent|
+  #    begin
+
+  #      skip1 = skip2 = false
+  #      while block = stream.read(1024)
+
+  #        begin 
+  #          stream_in1.write block; 
+  #        rescue IOError
+  #          Log.medium("Tee stream 1 #{Misc.fingerprint stream} IOError: #{$!.message}");
+  #          skip1 = true
+  #        end unless skip1 
+
+  #        begin 
+  #          stream_in2.write block
+  #        rescue IOError
+  #          Log.medium("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
+  #          skip2 = true
+  #        end unless skip2 
+
+  #      end
+
+  #      stream_in1.close unless stream_in1.closed?
+  #      stream.join if stream.respond_to? :join
+  #      stream_in2.close unless stream_in2.closed?
+  #    rescue Aborted, Interrupt
+  #      stream_out1.abort if stream_out1.respond_to? :abort
+  #      stream.abort if stream.respond_to? :abort
+  #      stream_out2.abort if stream_out2.respond_to? :abort
+  #      Log.medium "Tee aborting #{Misc.fingerprint stream}"
+  #      raise $!
+  #    rescue Exception
+  #      stream_out1.abort if stream_out1.respond_to? :abort
+  #      stream.abort if stream.respond_to? :abort
+  #      stream_out2.abort if stream_out2.respond_to? :abort
+  #      Log.medium "Tee exception #{Misc.fingerprint stream}"
+  #      raise $!
+  #    end
+  #  end
+
+  #  ConcurrentStream.setup stream_out1, :threads => splitter_thread
+  #  ConcurrentStream.setup stream_out2, :threads => splitter_thread
+
+  #  [stream_out1, stream_out2]
+  #end
+
+  def self.tee_stream_thread_multiple(stream, num = 2)
+    in_pipes = []
+    out_pipes = []
+    num.times do 
+      sout, sin = Misc.pipe
+      in_pipes << sin
+      out_pipes << sout
+    end
 
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
 
-        skip1 = skip2 = false
-        while block = stream.read(1024)
-
-          begin 
-            stream_in1.write block; 
-          rescue IOError
-            Log.medium("Tee stream 1 #{Misc.fingerprint stream} IOError: #{$!.message}");
-            skip1 = true
-          end unless skip1 
-
-          begin 
-            stream_in2.write block
-          rescue IOError
-            Log.medium("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
-            skip2 = true
-          end unless skip2 
-
+        skip = [false] * num
+        begin
+          while block = stream.readpartial(BLOCK_SIZE)
+
+            in_pipes.each_with_index do |sin,i|
+              begin 
+                sin.write block; 
+              rescue IOError
+                Log.medium("Tee stream #{i} #{Misc.fingerprint stream} IOError: #{$!.message}");
+                skip[i] = true
+              end unless skip[i] 
+            end
+        end
+        rescue IOError
         end
 
-        stream_in1.close unless stream_in1.closed?
+        in_pipes.each do |sin|
+          sin.close unless sin.closed?
+        end
         stream.join if stream.respond_to? :join
-        stream_in2.close unless stream_in2.closed?
       rescue Aborted, Interrupt
-        stream_out1.abort if stream_out1.respond_to? :abort
         stream.abort if stream.respond_to? :abort
-        stream_out2.abort if stream_out2.respond_to? :abort
+        out_pipes.each do |sout|
+          sout.abort if sout.respond_to? :abort
+        end
         Log.medium "Tee aborting #{Misc.fingerprint stream}"
         raise $!
       rescue Exception
-        stream_out1.abort if stream_out1.respond_to? :abort
         stream.abort if stream.respond_to? :abort
-        stream_out2.abort if stream_out2.respond_to? :abort
+        out_pipes.each do |sout|
+          sout.abort if sout.respond_to? :abort
+        end
         Log.medium "Tee exception #{Misc.fingerprint stream}"
         raise $!
       end
     end
 
-    ConcurrentStream.setup stream_out1, :threads => splitter_thread
-    ConcurrentStream.setup stream_out2, :threads => splitter_thread
+    out_pipes.each do |sout|
+      ConcurrentStream.setup sout, :threads => splitter_thread
+    end
 
-    #stream_out1.callback = stream.callback if stream.respond_to? :callback
-    #stream_out1.abort_callback = stream.abort_callback if stream.respond_to? :abort_callback
+    out_pipes
+  end
 
-    #stream_out2.callback = stream.callback if stream.respond_to? :callback
-    #stream_out2.abort_callback = stream.abort_callback if stream.respond_to? :abort_callback
+  def self.tee_stream_thread(stream)
+    tee_stream_thread_multiple(stream, 2)
+  end
 
-    [stream_out1, stream_out2]
+  def self.dup_stream_multiple(stream, num = 1)
+    stream_dup = stream.dup
+    if stream.respond_to? :annotate
+      stream.annotate stream_dup
+      stream.clear
+    end
+    tee1, *rest = Misc.tee_stream stream_dup, num + 1
+    stream.reopen(tee1)
+    rest
   end
 
   class << self
-    alias tee_stream tee_stream_thread 
+    alias tee_stream tee_stream_thread_multiple
+    alias dup_stream dup_stream_multiple
   end
 
   def self.read_full_stream(io)
     str = ""
     begin
-      while block = io.read(2048)
+      while block = io.read(BLOCK_SIZE)
         str << block
       end
       io.join if io.respond_to? :join
@@ -182,10 +250,17 @@ module Misc
       begin
         into = into.find if Path === into
         into = Open.open(into, :mode => 'w') if String === into 
-        into.sync == true if IO === into
-        while not io.closed? and block = io.read(2048)
-          into << block if into
+        into.sync = true if IO === into
+        into_close = false unless into.respond_to? :close
+        io.sync = true
+
+        begin
+          while block = io.readpartial(BLOCK_SIZE)
+            into << block if into
+          end
+        rescue EOFError
         end
+
         io.join if io.respond_to? :join
         io.close unless io.closed?
         into.close if into and into_close and not into.closed?
@@ -256,7 +331,7 @@ module Misc
 
             Open.write(tmp_path) do |f|
               f.sync = true
-              while block = content.read(2048)
+              while block = content.read(BLOCK_SIZE)
                 f.write block
               end
             end
@@ -340,7 +415,7 @@ module Misc
 
         sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
 
-        while block = sorted.read(2048)
+        while block = sorted.read(BLOCK_SIZE)
           sin.write block
         end
 
@@ -476,17 +551,6 @@ module Misc
     end
   end
 
-  def self.dup_stream(stream)
-    stream_dup = stream.dup
-    if stream.respond_to? :annotate
-      stream.annotate stream_dup
-      stream.clear
-    end
-    tee1, tee2 = Misc.tee_stream stream_dup
-    stream.reopen(tee1)
-    tee2
-  end
-
   def self.save_stream(file, stream)
     out, save = Misc.tee_stream stream
     out.filename = file