rbbt-util 5.21.38 → 5.21.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd8af3e8cd66f4c08804641788f1ae6384343f09
-  data.tar.gz: ee6b16cea96ef6699bc8f9b1fa359c6af2bc6d5a
+  metadata.gz: 97680be71c7f39a7c72f307cf0fa154ef8fbb524
+  data.tar.gz: 50b7802d5af30c133c85092bef090eff33ae44bd
 SHA512:
-  metadata.gz: ca555f21bd75ca53a3c74c92fa9e611dbbad7c09249c8d4bd3b041e0c498e7712fbe28384f3d397708cadda1efe4231e416d5e13b28fb313260b9de5ff1abecc
-  data.tar.gz: 5ea4b7dd1719264f3e9688aa8655b4ecee425306a5181e67de03b30ee592982c2217e3a7ed9f3f0203d0cb7b1265024a246ee2420c3ea3e77df81a447dbacea2
+  metadata.gz: 41a0673225580d3bb28ab960edcdf85a5a5564ddf044ff7445e3267cf3697a7af0d0d2b0b4b10214a489f8fdf16d2055e35bf4ef17dd644744af9c857ba41716
+  data.tar.gz: 9b5211a011ac1fc4bf7c338a372781d41e2471679802cad0e4ca4047a7a498344922a088ad957bb606c867e8ba1f625587beea688ddc0bbfa211a907ca052107

data/lib/rbbt/tsv/util.rb

@@ -93,13 +93,15 @@ module TSV
       file.open(open_options)
     when (defined? Tempfile and Tempfile)
       begin
-        file.rewind if file.respond_to?(:rewind) and file.eof?
-      rescue
+        pos = file.pos
+        file.rewind if file.respond_to?(:rewind) and pos != 0
+      rescue Exception
       end
       file
     when IO, StringIO, File
       begin
-        file.rewind if file.respond_to?(:rewind) and file.eof?
+        pos = file.pos
+        file.rewind if file.respond_to?(:rewind) and pos != 0
       rescue
       end
       file

data/lib/rbbt/util/cmd.rb

@@ -53,28 +53,31 @@ module CMD
 
     pid = fork {
       begin
+        Misc.purge_pipes(sin.last,sout.last,serr.last)
+
         sin.last.close
         sout.first.close
         serr.first.close
 
-        io = in_content
-        while IO === io
-          io.join if io.respond_to?(:join) and not io.joined?
-          io.close if io.respond_to?(:close) and not io.closed?
-          io = nil
+        if IO === in_content
+          in_content.close if in_content.respond_to?(:close) and not in_content.closed?
+          in_content.join if in_content.respond_to?(:join) and not in_content.joined?
         end
 
-        STDIN.reopen sin.first
-        sin.first.close
 
         STDERR.reopen serr.last
         serr.last.close
 
+        STDIN.reopen sin.first
+        sin.first.close
+
         STDOUT.reopen sout.last
         sout.last.close
 
+
         STDOUT.sync = STDERR.sync = true
-        
+
+
         exec(ENV, cmd)
 
         exit(-1)
@@ -89,32 +92,38 @@ module CMD
     sout.last.close
     serr.last.close
 
+
     sin = sin.last
     sout = sout.first
     serr = serr.first
-    
+
 
     Log.debug{"CMD: [#{pid}] #{cmd}" if log}
 
     if in_content.respond_to?(:read)
-      Thread.new do
+      in_thread = Thread.new do |parent|
         begin
-          loop do
-            break if in_content.closed?
-            block = in_content.read Misc::BLOCK_SIZE
-            break if block.nil? or block.empty?
-            sin.write block
+          begin
+            while c = in_content.readpartial(Misc::BLOCK_SIZE)
+              sin << c 
+            end
+          rescue EOFError
           end
+          sin.close  unless sin.closed?
 
-          sin.close unless sin.closed?
-          in_content.join if in_content.respond_to? :join and not dont_close_in
-          in_content.close unless in_content.closed? or dont_close_in
+          unless dont_close_in
+            in_content.close unless in_content.closed? 
+            in_content.join if in_content.respond_to? :join 
+          end
         rescue
+          parent.raise $!
           Process.kill "INT", pid
-          raise $!
+        ensure
+          sin.close  unless sin.closed?
         end
       end
     else
+      in_thread = nil
       sin.close
     end
 
@@ -124,15 +133,16 @@ module CMD
       pids = [pid]
     end
 
+
     if pipe
-      Thread.new do
+      err_thread = Thread.new do
         while line = serr.gets
           Log.log line, stderr if Integer === stderr and log
         end
         serr.close
       end
 
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
 
       sout
     else
@@ -141,23 +151,23 @@ module CMD
         while not serr.eof?
           err << serr.gets if Integer === stderr
         end
-        serr.close
-      end
+          serr.close
+        end
 
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
+        ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
 
-      out = StringIO.new sout.read
-      sout.close unless sout.closed?
+        out = StringIO.new sout.read
+        sout.close unless sout.closed?
 
-      Process.waitpid pid
+        Process.waitpid pid
 
-      if not $?.success? and not no_fail
-        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
-      else
-        Log.log err, stderr if Integer === stderr and log
-      end
+        if not $?.success? and not no_fail
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
+        else
+          Log.log err, stderr if Integer === stderr and log
+        end
 
-      out
+        out
     end
   end
 end

data/lib/rbbt/util/misc/pipes.rb

@@ -260,6 +260,7 @@ module Misc
   def self.consume_stream(io, in_thread = false, into = nil, into_close = true, &block)
     return if Path === io
     return unless io.respond_to? :read 
+
     if io.respond_to? :closed? and io.closed?
       io.join if io.respond_to? :join
       return
@@ -426,25 +427,13 @@ module Misc
         end
 
         line_stream = Misc.open_pipe do |line_stream_in|
-          begin
-            while line
-              line_stream_in.puts line
-              line = stream.gets
-            end
-            stream.join if stream.respond_to? :join
-          rescue
-            stream.abort if stream.respond_to? :abort
-            raise $!
-          end
+          line_stream_in.puts line
+          Misc.consume_stream(stream, false, line_stream_in)
         end
 
         sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
 
-        while block = sorted.read(BLOCK_SIZE)
-          sin.write block
-        end
-
-        sorted.join if sorted.respond_to? :join
+        Misc.consume_stream(sorted, false, sin)
       rescue
         if defined? step and step
           step.abort
@@ -626,7 +615,7 @@ module Misc
       else
         file1 = TmpFile.tmp_file
         erase << file1
-        Open.write(file1, TSV.get_stream(stream1))
+        Misc.consume_stream(TSV.get_stream(stream1), false, file1)
       end
 
       if Path === stream2 or (String === stream2 and File.exist? stream2)
@@ -634,7 +623,7 @@ module Misc
       else
         file2 = TmpFile.tmp_file
         erase << file2
-        Open.write(file2, TSV.get_stream(stream2))
+        Misc.consume_stream(TSV.get_stream(stream2), false, file2)
       end
 
       CMD.cmd("env LC_ALL=C comm #{args} '#{file1}' '#{file2}'", :pipe => true, :post => Proc.new{ erase.each{|f| FileUtils.rm f } }) 

data/test/rbbt/util/misc/test_pipes.rb

@@ -107,6 +107,22 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
+  def test_sort_stream2
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+    s = StringIO.new text
+    sorted = Misc.sort_stream(Misc.sort_stream(s))
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
   def test_dup_stream
     text =<<-EOF
 #: :sep=" "
@@ -201,6 +217,7 @@ line4
     text2 =<<-EOF
 line3
 line1
+line5
     EOF
 
     Log.severity = 0

data/test/rbbt/util/test_cmd.rb

@@ -53,11 +53,13 @@ line33
     EOF
 
     TmpFile.with_file(text * 100) do |file|
-      CMD.cmd("gzip #{ file }")
 
-      gz = CMD.cmd("gunzip", :in => File.open(file + '.gz'), :pipe => true)
-      io = CMD.cmd('tail -n 10', :in => gz, :pipe => true)
-      assert_equal 10, io.read.split(/\n/).length
+      Open.open(file) do |f|
+        io = CMD.cmd('tail -n 10', :in => f, :pipe => true)
+        io2 = CMD.cmd('head -n 10', :in => io, :pipe => true)
+        io3 = CMD.cmd('head -n 10', :in => io2, :pipe => true)
+        assert_equal 10, io3.read.split(/\n/).length
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.38
+  version: 5.21.39
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-03 00:00:00.000000000 Z
+date: 2016-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake