rbbt-util 5.13.1 → 5.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a03adabca2810fb92c9771c98c6cee6126fec535
-  data.tar.gz: 173c80fd823140c2d7cb5443ae034fa61d89b74a
+  metadata.gz: 52c2b4f2b6f065da491646388575b22ffccddee4
+  data.tar.gz: 48c8543fb121288f15a998bf4ce0237e56bdc2a8
 SHA512:
-  metadata.gz: 43fa5b928dd62ddb3627d44255ec4497eba7ad61f2d16236df36e2cdb1f2243182b92ca4e11c4c279f2db6ea91d68e514530f9ff64abbc27a6ea93b2c145acf6
-  data.tar.gz: 4c9c7b955f6157b9ed7c0640454be5e92eef62c2e831bcfe1ea1fc16621d6c80c5bf48f72cf0233e8213582a06fd9ec58ac87005b1f1ef7d7f3fc21572352b84
+  metadata.gz: b218fc2960eb7f021d8628fa08b5363362efdc6bbf4b04ea1002140dcdf93dd7556f39f6265d77c8d35c3c234a0ecedcfe94197499ea080c663249d748ef963e
+  data.tar.gz: 3562d3fb1ccbf97705e4923642814b2c4519942c89dcfc97d9cc74802a220e32c338f2bfce65d689dffc0b35e99982f817236a75b67a0ebc8faeda7b5b516024

data/lib/rbbt/persist.rb

@@ -116,7 +116,6 @@ module Persist
             begin
               Marshal.load(stream) 
             rescue
-              Log.exception $!
               raise $!
             end
           else
@@ -135,7 +134,7 @@ module Persist
         raise "Unknown persistence: #{ type }"
       end
     rescue
-      Log.error "Exception loading #{ type } #{ path }: #{$!.message}"
+      Log.warn "Exception loading #{ type } #{ path }: #{$!.message}"
       raise $!
     end
   end
@@ -191,30 +190,6 @@ module Persist
     end
   end
 
-  #def self.tee_stream_fork(stream, path, type, callback = nil)
-  #  file, out = Misc.tee_stream(stream)
-
-  #  saver_pid = Process.fork do
-  #    out.close
-  #    stream.close
-  #    Misc.purge_pipes
-  #    begin
-  #      Misc.lock(path) do
-  #        save_file(path, type, file)
-  #      end
-  #    rescue Aborted
-  #      stream.abort if stream.respond_to? :abort
-  #      raise $!
-  #    rescue Exception
-  #      Log.exception $!
-  #      Kernel.exit! -1
-  #    end
-  #    Kernel.exit! 0
-  #  end
-  #  file.close
-  #  ConcurrentStream.setup(out, :pids => [saver_pid], :filename => path)
-  #end
-
   def self.tee_stream_thread(stream, path, type, callback = nil)
     file, out = Misc.tee_stream(stream)
 
@@ -225,11 +200,11 @@ module Persist
           save_file(path, type, file)
         end
       rescue Aborted
-        Log.error "Persist stream thread aborted: #{ Log.color :blue, path }"
+        Log.warn "Persist stream thread aborted: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
+        parent.raise $!
       rescue Exception
-        Log.error "Persist stream thread exception: #{ Log.color :blue, path }"
-        Log.exception $!
+        Log.warn "Persist stream thread exception: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
         parent.raise $!
       end
@@ -250,10 +225,10 @@ module Persist
             end
             Log.high "Stream pipe saved: #{path}"
           rescue Aborted
-            Log.error "Persist stream pipe exception: #{ Log.color :blue, path }"
+            Log.warn "Persist stream pipe exception: #{ Log.color :blue, path }"
             stream.abort if stream.respond_to? :abort
           rescue Exception
-            Log.error "Persist stream pipe exception: #{ Log.color :blue, path }"
+            Log.warn "Persist stream pipe exception: #{ Log.color :blue, path }"
             Log.exception $!
             stream.abort if stream.respond_to? :abort
             parent.raise $!
@@ -266,7 +241,7 @@ module Persist
               sin.write block
             end
           rescue Aborted
-            Log.error "Tee stream thread aborted"
+            Log.warn "Tee stream thread aborted"
             sout.abort if sout.respond_to? :abort
             sin.abort if sin.respond_to? :abort
           rescue Exception
@@ -328,7 +303,7 @@ module Persist
         res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
         ConcurrentStream.setup res do
           begin
-            lockfile.unlock
+            lockfile.unlock if lockfile.locked?
           rescue
             Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
@@ -336,7 +311,7 @@ module Persist
         end
         res.abort_callback = Proc.new do
           begin
-            lockfile.unlock
+            lockfile.unlock if lockfile.locked?
           rescue
             Log.exception $!
             Log.warn "Lockfile exception: " << $!.message
@@ -406,7 +381,7 @@ module Persist
       end
 
     rescue
-      Log.error "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
+      Log.warn "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
       FileUtils.rm path if Open.exists? path 
       raise $!
     end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -14,13 +14,13 @@ module TSV
     filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
     filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
     stream_obj = obj_stream(obj) || obj
-    filename_obj.nil? ? Misc.fingerprint(stream_obj) : filename_obj + "(#{Misc.fingerprint(stream_obj)})"
+    obj.class.to_s << "-" << Misc.fingerprint(stream_obj)
   end
 
   def self.report(msg, obj, into)
     into = into[:into] if Hash === into and into.include? :into
 
-    Log.error "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
+    Log.medium "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
   end
 
   #{{{ TRAVERSE OBJECTS
@@ -96,13 +96,20 @@ module TSV
 
   def self.traverse_io_array(io, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
+    if File === io and io.closed? 
+      begin
+        Log.medium "Rewinding stream #{stream_name(io)}"
+        io.reopen io.filename, "r"
+      rescue
+        Log.exception $!
+        raise "File closed and could not reopen #{stream_name(io)}"
+      end
+    end
+
     if callback
       while line = io.gets
-        begin
-          callback.call yield line.strip
-        ensure
-          bar.tick if bar
-        end
+        callback.call yield line.strip
+        bar.tick if bar
       end
     else
       while line = io.gets
@@ -114,17 +121,22 @@ module TSV
 
   def self.traverse_io(io, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
-    begin
-      if callback
-        TSV::Parser.traverse(io, options) do |k,v|
-          callback.call yield k, v
-        end
-      else
-        TSV::Parser.traverse(io, options, &block)
+    if File === io and io.closed? 
+      begin
+        Log.medium "Rewinding stream #{stream_name(io)}"
+        io.reopen io.filename, "r"
+      rescue
+        Log.exception $!
+        raise "File closed and could not reopen #{stream_name(io)}"
       end
-    rescue
-      Log.error "Traverse IO error"
-      raise $!
+    end
+
+    if callback
+      TSV::Parser.traverse(io, options) do |k,v|
+        callback.call yield k, v
+      end
+    else
+      TSV::Parser.traverse(io, options, &block)
     end
     join.call if join
   end
@@ -135,64 +147,90 @@ module TSV
       options[:type] = :single
     end
 
-    case obj
-    when TSV
-      traverse_tsv(obj, options, &block)
-    when Hash
-      traverse_hash(obj, options, &block)
-    when TSV::Parser
-      callback = Misc.process_options options, :callback
-      if callback
-        obj.traverse(options) do |k,v|
-          callback.call yield k, v
-        end
-      else
-        obj.traverse(options, &block)
-      end
-    when IO, File
-      begin
-        if options[:type] == :array
-          traverse_io_array(obj, options, &block)
+    Log.medium "Traversing #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
+    begin
+      sleep 1
+      case obj
+      when TSV
+        traverse_tsv(obj, options, &block)
+      when Hash
+        traverse_hash(obj, options, &block)
+      when TSV::Parser
+        callback = Misc.process_options options, :callback
+        if callback
+          obj.traverse(options) do |k,v|
+            callback.call yield k, v
+          end
         else
-          traverse_io(obj, options, &block)
+          obj.traverse(options, &block)
         end
-      rescue Exception
-        obj.abort if obj.respond_to? :abort
-        raise $!
-      ensure
-        obj.close if obj.respond_to? :close and not obj.closed?
-        obj.join if obj.respond_to? :join
-      end
-    when Path
-      obj.open do |stream|
-        traverse_obj(stream, options, &block)
-      end
-    when TSV::Dumper
-      traverse_obj(obj.stream, options, &block)
-    when (defined? Step and Step)
+      when IO, File
+        begin
+          if options[:type] == :array
+            traverse_io_array(obj, options, &block)
+          else
+            traverse_io(obj, options, &block)
+          end
+        rescue Aborted
+          obj.abort if obj.respond_to? :abort
+          raise $!
+        rescue Exception
+          obj.abort if obj.respond_to? :abort
+          raise $!
+        ensure
+          obj.close if obj.respond_to? :close and not obj.closed?
+          obj.join if obj.respond_to? :join
+        end
+      when Path
+        obj.open do |stream|
+          traverse_obj(stream, options, &block)
+        end
+      when TSV::Dumper
+        traverse_obj(obj.stream, options, &block)
+      when (defined? Step and Step)
 
-      stream = obj.get_stream
+        stream = obj.get_stream
 
-      if stream
-        traverse_obj(stream, options, &block)
-      else
-        obj.join
-        traverse_obj(obj.path, options, &block)
-      end
-    when Array
-      traverse_array(obj, options, &block)
-    when String
-      if Open.remote? obj or Misc.is_filename? obj
-        Open.open(obj) do |s|
-          traverse_obj(s, options, &block)
+        if stream
+          traverse_obj(stream, options, &block)
+        else
+          obj.join
+          traverse_obj(obj.path, options, &block)
         end
+      when Array
+        traverse_array(obj, options, &block)
+      when String
+        if Open.remote? obj or Misc.is_filename? obj
+          Open.open(obj) do |s|
+            traverse_obj(s, options, &block)
+          end
+        else
+          raise "Can not open obj for traversal #{Misc.fingerprint obj}"
+        end
+      when nil
+        raise "Can not traverse nil object into #{stream_name(options[:into])}"
       else
-        raise "Can not open obj for traversal #{Misc.fingerprint obj}"
+        raise "Unknown object for traversal: #{Misc.fingerprint obj }"
       end
-    when nil
-      raise "Can not traverse nil object into #{stream_name(options)}"
-    else
-      raise "Unknown object for traversal: #{Misc.fingerprint obj }"
+    rescue IOError
+      Log.warn "IOError traversing #{stream_name(obj)}: #{$!.message}"
+      stream = obj_stream(obj)
+      stream.abort if stream and stream.respond_to? :abort
+      raise $!
+    rescue Errno::EPIPE
+      Log.warn "Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"
+      raise $!
+    rescue Aborted
+      Log.warn "Aborted traversing #{stream_name(obj)}"
+      stream = obj_stream(obj)
+      stream.abort if stream and stream.respond_to? :abort
+      raise $!
+    rescue Exception
+      Log.warn "Exception traversing #{stream_name(obj)}"
+      Log.exception $!
+      stream = obj_stream(obj)
+      stream.abort if stream and stream.respond_to? :abort
+      raise $!
     end
   end
 
@@ -236,9 +274,13 @@ module TSV
         end
       end
 
-      thread.join
+      begin
+        thread.join
+      rescue 
+        raise $!
+      end
     rescue Interrupt, Aborted
-      Log.error "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
+      Log.warn "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
@@ -246,9 +288,7 @@ module TSV
       q.abort
       raise $!
     rescue Exception
-      Log.error "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
-      Log.exception $!
-
+      Log.warn "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}"
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
@@ -350,11 +390,12 @@ module TSV
       begin
         traverse_run(obj, threads, cpus, options, &block)
         into.close if into.respond_to? :close
-      rescue Exception
-        Log.exception $!
-        parent.raise $!
+      rescue
+        stream = obj_stream(obj)
+        stream.abort if stream and stream.respond_to? :abort
         stream = obj_stream(into)
         stream.abort if stream and stream.respond_to? :abort
+        parent.raise $!
       end
     end
     ConcurrentStream.setup(obj_stream(into), :threads => thread)
@@ -398,7 +439,7 @@ module TSV
 
     if into
       bar = Misc.process_options options, :bar
-          
+
       options[:join] = Proc.new do
         Log::ProgressBar.remove_bar(bar)
       end if bar
@@ -407,7 +448,6 @@ module TSV
         begin
           store_into into, e
         rescue Aborted
-          Log.error "Traversal info #{stream_name into} aborted"
           raise $!
         rescue Exception
           Log.exception $!

data/lib/rbbt/tsv/parser.rb

@@ -134,7 +134,6 @@ module TSV
         return parts.shift.split(@sep2, -1).first, parts.collect{|value| value.split(@sep2, -1)}.flatten if 
         field_positions.nil? and (key_position.nil? or key_position == 0)
       rescue
-        eee [:rescue, orig]
         raise $!
       end
 
@@ -536,11 +535,14 @@ module TSV
             end
           rescue END_PARSING
             break
-          #rescue IOError
-          #  Log.exception $!
-          #  break
+          rescue Errno::EPIPE
+            Log.error "Pipe closed while parsing #{Misc.fingerprint stream}: #{$!.message}"
+            raise $!
           rescue Exception
+            Log.error "Exception parsing #{Misc.fingerprint stream}: #{$!.message}"
+            Log.exception $!
             stream.abort if stream.respond_to? :abort
+            stream.join if stream.respond_to? :join
             raise $!
           end
         end

data/lib/rbbt/tsv/stream.rb

@@ -28,8 +28,8 @@ module TSV
     input_options = []
 
     input_source_streams = inputs.collect do |input|
-      stream = TSV.get_stream input
-      stream = sort ? Misc.sort_stream(stream) : stream
+      stream = sort ? Misc.sort_stream(input) : TSV.get_stream(input)
+      stream
     end
 
     input_source_streams.each do |stream|
@@ -52,4 +52,109 @@ module TSV
     dumper.stream = Misc.paste_streams input_streams, input_lines, options[:sep], header
     dumper
   end
+
+  def self.paste_streams(streams, options = {})
+    options = Misc.add_defaults options, :sep => "\t", :sort => true
+    sort, sep = Misc.process_options options, :sort, :sep
+
+    Misc.open_pipe do |sin|
+      num_streams = streams.length
+
+      streams = streams.collect do |stream|
+        if defined? Step and Step === stream
+          stream.get_stream || stream.join.path.open
+        else
+          stream
+        end
+      end
+
+      streams = streams.collect do |stream|
+        Misc.sort_stream(stream)
+      end if sort
+
+      lines = []
+      fields = []
+      key_fields = []
+      input_options = []
+      empty = []
+
+      streams = streams.collect do |stream|
+        parser = TSV::Parser.new stream, options
+        lines << parser.first_line
+        empty << stream if parser.first_line.nil?
+        key_fields << parser.key_field
+        fields << parser.fields
+        input_options << parser.options
+
+        parser.stream
+      end
+
+      key_field = key_fields.compact.first
+      fields = fields.compact.flatten
+      options = options.merge(input_options.first)
+
+      sin.puts TSV.header_lines(key_field, fields, options)
+
+      empty.each do |stream|
+        i = streams.index stream
+        lines.delete_at i
+        fields.delete_at i
+        key_fields.delete_at i
+        input_options.delete_at i
+      end
+
+      begin
+        done_streams = []
+
+        keys = []
+        parts = []
+        lines.each_with_index do |line,i|
+          key, *p = line.strip.split(sep, -1) 
+          keys[i] = key
+          parts[i] = p
+        end
+        sizes = parts.collect{|p| p.length }
+        last_min = nil
+        while lines.compact.any?
+          min = keys.compact.sort.first
+          str = []
+          keys.each_with_index do |key,i|
+            case key
+            when min
+              str << [parts[i] * sep]
+              line = lines[i] = begin
+                                  streams[i].gets
+                                rescue
+                                  Log.exception $!
+                                  nil
+                                end
+              if line.nil?
+                stream = streams[i]
+                #stream.join if stream.respond_to? :join
+                keys[i] = nil
+                parts[i] = nil
+              else
+                k, *p = line.strip.split(sep, -1)
+                keys[i] = k
+                parts[i] = p
+              end
+            else
+              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+            end
+          end
+
+          sin.puts [min, str*sep] * sep
+        end
+        streams.each do |stream|
+          #stream.join if stream.respond_to? :join
+        end
+      rescue 
+        Log.exception $!
+        streams.each do |stream|
+          stream.abort if stream.respond_to? :abort
+        end
+        raise $!
+      end
+    end
+  end
 end