rbbt-util 5.13.34 → 5.13.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1581db188b182a19a32b933fa09143b05a63f673
-  data.tar.gz: 4ff0988f92691a719fa4e5d255f5e91bde236ac5
+  metadata.gz: 81458a4ac4aaf570ca085039bc9edef474939dcf
+  data.tar.gz: 0347d3cdb338145d6633126373eaad1430aaea79
 SHA512:
-  metadata.gz: b0b93bb3972edb714994d0e5086f8d4e6d469cd719de7ab453fc36dc5f4b4a9bd98b80028f918ba1beb2d916678af9d467492ec8767477d874fcc8dc589bc651
-  data.tar.gz: 3e46304432334c7fab8b8e95425794cec108b8dd80c4e598e5a7ad017eb31cd70c5a202997785ca1178a32dbb4fd839f50a2b4ca980b91f0ad1f8b0a656e5105
+  metadata.gz: f3eed434127735a837685ffd0ceb170c4e05988cdb02cab6626d63b1c7f4968f62ceff494bf50a18ad10baef182a9cea2b4f135b2fef21814d472aaf7b7ffd73
+  data.tar.gz: 6c212fb6f2cd6a28ff2e45a3fcceffccc571a02f802f45627c0b7299b2df38461eb8181434ac7a21caf83b9c1855a2d51d1fdb9f7a9b27501bcc5b84b3481232

data/lib/rbbt/tsv/stream.rb

@@ -62,6 +62,7 @@ module TSV
 
       streams = streams.collect do |stream|
         if defined? Step and Step === stream
+          stream.grace
           stream.get_stream || stream.join.path.open
         else
           stream
@@ -69,7 +70,10 @@ module TSV
       end
 
       streams = streams.collect do |stream|
-        Misc.sort_stream(stream)
+        sorted = Misc.sort_stream(stream)
+        stream.annotate sorted if stream.respond_to? :annotate
+        stream.clear if stream.respond_to? :annotate
+        sorted
       end if sort
 
       lines = []
@@ -123,6 +127,7 @@ module TSV
             case key
             when min
               str << [parts[i] * sep]
+
               line = lines[i] = begin
                                   streams[i].gets
                                 rescue
@@ -131,7 +136,6 @@ module TSV
                                 end
               if line.nil?
                 stream = streams[i]
-                stream.join if stream.respond_to? :join
                 keys[i] = nil
                 parts[i] = nil
               else
@@ -150,12 +154,10 @@ module TSV
           stream.join if stream.respond_to? :join
         end
       rescue Exception
-        ts = streams.collect do |stream|
-          Thread.new do
-            stream.abort if stream.respond_to? :abort
-          end
+        Log.error "Exception pasting streams #{streams.inspect}: #{$!.message}"
+        streams.each do |stream|
+          stream.abort
         end
-        ts.each do |t| t.join end
         raise $!
       end
     end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -36,10 +36,11 @@ module ConcurrentStream
     stream.filename = filename
     stream.autojoin = autojoin
     stream.joined = joined
+    stream
   end
 
   def clear
-    threads, pids, callback, abort_callback = nil
+    threads, pids, callback, abort_callback, joined = nil
   end
 
   def joined?

data/lib/rbbt/util/misc/pipes.rb

@@ -90,6 +90,10 @@ module Misc
     stream_out1, stream_in1 = Misc.pipe
     stream_out2, stream_in2 = Misc.pipe
 
+    if ConcurrentStream === stream 
+      stream.annotate stream_out1
+    end
+
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
         skip1 = skip2 = false
@@ -109,20 +113,19 @@ module Misc
           end unless skip2 
         end
         stream_in1.close unless stream_in1.closed?
-        stream_in2.close unless stream_in2.closed?
         stream.join if stream.respond_to? :join
+        stream_in2.close unless stream_in2.closed?
       rescue Aborted, Interrupt
-        Log.medium "Tee aborting #{Misc.fingerprint stream}"
-        stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
+        stream.abort if stream.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
-        Log.medium "tee_stream_thread aborted: #{$!.message}"
+        Log.medium "Tee aborting #{Misc.fingerprint stream}"
+        raise $!
       rescue Exception
-        stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
+        stream.abort if stream.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
-        stream.join if stream.respond_to? :join
-        Log.medium "Exception in tee_stream_thread: #{$!.message}"
+        Log.medium "Tee exception #{Misc.fingerprint stream}"
         raise $!
       end
     end
@@ -228,19 +231,12 @@ module Misc
             File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
 
-            out = nil
-            PIPE_MUTEX.synchronize do
-              out = File.open(tmp_path, 'wb') 
-              out.sync = true
-              OPEN_PIPE_IN << out
+            Open.write(tmp_path) do |f|
+              f.sync = true
+              while block = content.read(2048)
+                f.write block
+              end
             end
-
-            while block = content.read(2048); 
-              out.write block
-            end  
-
-            content.close
-            out.close
           else
             File.open(tmp_path, 'wb') do |f|  end
           end

data/lib/rbbt/util/open.rb

@@ -237,7 +237,10 @@ module Open
     dir_sub_path_source = find_repo_dir(source)
     dir_sub_path_target = find_repo_dir(target)
 
-    return FileUtils.mv source, target if dir_sub_path_source.nil? and dir_sub_path_target.nil?
+    if dir_sub_path_source.nil? and dir_sub_path_target.nil?
+      FileUtils.mkdir_p File.dirname(target) unless File.exists? File.dirname(target)
+      return FileUtils.mv source, target 
+    end
 
     if dir_sub_path_source.nil?
       save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source))

data/lib/rbbt/workflow/accessor.rb

@@ -29,6 +29,17 @@ class Step
     path.nil? ? nil : path + '.info'
   end
 
+  def self.step_info(path)
+    begin
+      Open.open(info_file(path)) do |f|
+        INFO_SERIALIAZER.load(f)
+      end
+    rescue Exception
+      Log.exception $!
+      {}
+    end
+  end
+
   def self.job_name_for_info_file(info_file, extension = nil)
     if extension and not extension.empty?
       info_file.sub(/\.#{extension}\.info$/,'')

data/lib/rbbt/workflow/step/run.rb

@@ -400,8 +400,7 @@ class Step
     stream = get_stream if @result
     if stream
       begin
-        @stream_data = StringIO.new
-        Misc.consume_stream stream, false, @stream_data
+        Misc.consume_stream stream 
       rescue Exception
         self._abort
         raise $!
@@ -434,9 +433,6 @@ class Step
     begin
       if pid.nil? or Process.pid == pid
         dependencies.each{|dep| dep.join }
-        while not done?
-          sleep 1
-        end
       else
         begin
           Log.debug{"Waiting for pid: #{pid}"}

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -339,7 +339,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_raise ProcessFailed do
       TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
         k = k.first
-        raise StopException if rand(100) < 20
+        raise ProcessFailed if rand(100) < 20
         [k,v]
       end
       dumper.stream.join

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.34
+  version: 5.13.35
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-08 00:00:00.000000000 Z
+date: 2014-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake