rbbt-util 5.13.37 → 5.14.0

data/lib/rbbt/tsv/manipulate.rb

@@ -162,8 +162,7 @@ module TSV
         desc = @monitor[:desc] if @monitor.include? :desc 
         step = @monitor[:step] if @monitor.include? :step 
       end
-      #progress_monitor = Progress::Bar.new(size, 0, step, desc)
-      progress_monitor = Log::ProgressBar.new(size, :desc => desc)
+      progress_monitor = Log::ProgressBar.new_bar(size, :desc => desc)
     else
       progress_monitor = nil
     end
@@ -192,10 +191,10 @@ module TSV
         when :flat, :single
           prepare_entity(value, traverser.new_field_names.first, entity_options)
         end
-
       end
 
 
+
       if zipped
 
         keys.each_with_index do |k,i|
@@ -224,8 +223,11 @@ module TSV
         end
 
       end
+
     end
 
+    Log::ProgressBar.remove_bar progress_monitor if progress_monitor
+
     [traverser.new_key_field_name, traverser.new_field_names]
   end
 
@@ -415,7 +417,7 @@ module TSV
       case
       when (Array === method and (key == :key or key_field == key))
         with_unnamed do
-          Annotated.purge(method).uniq.each{|key| 
+          TSV.traverse(Annotated.purge(method).uniq, :bar => true){|key| 
             new[key] = self[key] if invert ^ (self.include? key)
           }
         end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -38,7 +38,6 @@ module TSV
         end
       end
     rescue Exception
-      Log.exception $!
       nil
     end
   end
@@ -186,8 +185,9 @@ module TSV
       end
     else
       options[:monitor] = bar
-      TSV::Parser.traverse(io, options, &block)
+      TSV::Parser.traverse(io, options.merge(:monitor => bar), &block)
     end
+    Log::ProgressBar.remove_bar(bar) if bar
     join.call if join
   end
 
@@ -213,7 +213,7 @@ module TSV
         else
           obj.traverse(options, &block)
         end
-      when IO, File
+      when IO, File, StringIO
         begin
           if options[:type] == :array
             traverse_io_array(obj, options, &block)
@@ -392,10 +392,11 @@ module TSV
       end 
       true
     rescue Aborted, Interrupt
-      Log.medium "Aborted storing into #{Misc.fingerprint store}: #{$!.message}"
+      Log.medium "Aborted storing into #{Misc.fingerprint store}"
       stream = obj_stream(store)
       stream.abort if stream.respond_to? :abort
     rescue Exception
+      Log.medium "Exception storing into #{Misc.fingerprint store}: #{$!.message}"
       stream = obj_stream(store)
       stream.abort if stream.respond_to? :abort
       raise $!
@@ -462,7 +463,7 @@ module TSV
     thread = Thread.new(Thread.current) do |parent|
       begin
         traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to? :close
+        into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?)
       rescue Exception
         stream = obj_stream(obj)
         stream.abort if stream and stream.respond_to? :abort
@@ -557,7 +558,7 @@ module TSV
         traverse_stream(obj, threads, cpus, options, &block)
       else
         traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to? :close
+        into.close if into.respond_to?(:close) and not (into.respond_to? :closed and into.closed?)
       end
 
       into

data/lib/rbbt/tsv/parser.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/cmd'
 module TSV
   class Parser
-    attr_accessor :stream, :filename, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream
+    attr_accessor :stream, :filename, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream, :preamble
 
     class SKIP_LINE < Exception; end
     class END_PARSING < Exception; end
@@ -13,20 +13,22 @@ module TSV
 
     def parse_header(stream)
       options = {}
+      @preamble = []
 
       # Get line
 
       #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
       line = stream.gets
-      raise "Empty content: #{ stream.inspect }" if line.nil?
-      line = Misc.fixutf8 line
-      line.chomp!
+      return {} if line.nil?
+      #raise "Empty content: #{ stream.inspect }" if line.nil?
+      line = Misc.fixutf8 line.chomp
 
       # Process options line
 
       if line and line =~ /^#{@header_hash}: (.*)/
-        options = Misc.string2hash $1.strip
-        line = Misc.fixutf8 stream.gets
+        options = Misc.string2hash $1.chomp
+        line = stream.gets
+        line = Misc.fixutf8 line.chomp if line
       end
 
       # Determine separator
@@ -35,16 +37,20 @@ module TSV
 
       # Process fields line
 
+      preamble << line if line
       while line and Misc.fixutf8(line) =~ /^#{@header_hash}/ 
-        line.chomp!
         @fields = line.split(@sep)
         @key_field = @fields.shift
         @key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
 
         #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
-        line = @header_hash != "" ?  Misc.fixutf8(stream.gets) : nil
+        line = (@header_hash != "" ?  stream.gets : nil)
+        line = Misc.fixutf8 line.chomp if line
+        preamble << line if line
       end
 
+      @preamble = preamble[0..-3] * "\n"
+
       line ||= stream.gets
 
       @first_line = line
@@ -112,7 +118,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| value.split(@sep2, -1)}
+               end.collect{|value| value.nil? ? [] : value.split(@sep2, -1) }
       [keys, values]
     end
 
@@ -482,6 +488,7 @@ module TSV
       # first line
       line = self.rescue_first_line
 
+      progress_monitor, monitor = monitor, nil if Log::ProgressBar === monitor
       # setup monitor
       if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
         size = case
@@ -497,8 +504,6 @@ module TSV
           step = monitor[:step] if monitor.include? :step 
         end
         progress_monitor = Log::ProgressBar.new(size, :desc => desc)
-      else
-        progress_monitor = nil
       end
 
       # parser 
@@ -507,7 +512,8 @@ module TSV
 
         while not line.nil? 
           begin
-            progress_monitor.tick(stream.pos) if progress_monitor 
+            #progress_monitor.tick(stream.pos) if progress_monitor 
+            progress_monitor.tick if progress_monitor 
 
             raise SKIP_LINE if line.empty?
 
@@ -520,8 +526,6 @@ module TSV
             
             yield key, values
 
-            #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
-
             line = stream.gets
 
             line_num += 1
@@ -541,13 +545,13 @@ module TSV
             raise $!
           rescue Exception
             Log.error "Exception parsing #{Misc.fingerprint stream}: #{$!.message}"
-            stream.abort if stream.respond_to? :abort
+            stream.abort $! if stream.respond_to? :abort
             raise $!
           end
         end
 
       ensure
-        stream.close
+        stream.close unless stream.closed?
         stream.join if stream.respond_to? :join
       end
 

data/lib/rbbt/tsv/stream.rb

@@ -1,4 +1,3 @@
-require 'rbbt/tsv/parser'
 require 'rbbt/tsv/dumper'
 module TSV
 
@@ -17,56 +16,26 @@ module TSV
     dumper
   end
  
-  def self.paste_streams(inputs, options = {})
-    options = Misc.add_defaults options, :sep => "\t", :sort => false
-    sort = Misc.process_options options, :sort
-
-    input_streams = []
-    input_lines = []
-    input_fields = []
-    input_key_fields = []
-    input_options = []
-
-    input_source_streams = inputs.collect do |input|
-      stream = sort ? Misc.sort_stream(input) : TSV.get_stream(input)
-      stream
-    end
+  def self.paste_streams(streams, options = {})
+    options = Misc.add_defaults options, :sep => "\t", :sort => true
+    sort, sep, preamble = Misc.process_options options, :sort, :sep, :preamble
 
-    input_source_streams.each do |stream|
-      parser = TSV::Parser.new stream, options
-      input_streams << parser.stream
-      input_lines << parser.first_line
-      input_fields << parser.fields
-      input_key_fields << parser.key_field
-      input_options << parser.options
-    end
 
-    key_field = input_key_fields.first
-    fields = input_fields.flatten
-    options = options.merge(input_options.first)
 
-    dumper = TSV::Dumper.new options.merge(:key_field => key_field, :fields => fields)
-    dumper.close_in
-    dumper.close_out
-    header = TSV.header_lines(key_field, fields, options)
-    dumper.stream = Misc.paste_streams input_streams, input_lines, options[:sep], header
-    dumper
-  end
+    out = Misc.open_pipe do |sin|
 
-  def self.paste_streams(streams, options = {})
-    options = Misc.add_defaults options, :sep => "\t", :sort => true
-    sort, sep = Misc.process_options options, :sort, :sep
-
-    streams = streams.collect do |stream|
-      if defined? Step and Step === stream
-        stream.grace
-        stream.get_stream || stream.join.path.open
-      else
-        stream
+      streams = streams.collect do |stream|
+        case stream
+        when (defined? Step and Step) 
+          stream.grace
+          stream.get_stream || stream.join.path.open
+        when Path
+          stream.open
+        else
+          stream
+        end
       end
-    end
 
-    out = Misc.open_pipe do |sin|
       num_streams = streams.length
 
       streams = streams.collect do |stream|
@@ -75,12 +44,13 @@ module TSV
         sorted
       end if sort
 
-      lines = []
-      fields = []
-      sizes = []
-      key_fields = []
+      lines         = []
+      fields        = []
+      sizes         = []
+      key_fields    = []
       input_options = []
-      empty = []
+      empty         = []
+      preambles     = []
 
       streams = streams.collect do |stream|
         parser = TSV::Parser.new stream, options
@@ -88,8 +58,9 @@ module TSV
         empty << stream if parser.first_line.nil?
         key_fields << parser.key_field
         fields << parser.fields
-        sizes << parser.fields.length
+        sizes << parser.fields.length if parser.fields
         input_options << parser.options
+        preambles << parser.preamble if TrueClass === preamble and not parser.preamble.empty?
 
         parser.stream
       end
@@ -98,12 +69,20 @@ module TSV
       fields = fields.compact.flatten
       options = options.merge(input_options.first)
 
-      sin.puts TSV.header_lines(key_field, fields, options)
+      preamble_txt = case preamble
+                     when TrueClass
+                       preambles * "\n"
+                     when String
+                       preamble
+                     else
+                       nil
+                     end
+
+      header = TSV.header_lines(key_field, fields, options.merge(:preamble => preamble_txt))
+      sin.puts header
 
       empty_pos = empty.collect{|stream| streams.index stream }
       empty_pos.sort.reverse.each do |i|
-        lines.delete_at i
-        fields.delete_at i
         key_fields.delete_at i
         input_options.delete_at i
       end
@@ -114,10 +93,18 @@ module TSV
         keys = []
         parts = []
         lines.each_with_index do |line,i|
-          key, *p = line.strip.split(sep, -1) 
-          keys[i] = key
-          parts[i] = p
+          if line.nil?
+            keys[i] = nil
+            parts[i] = nil
+          else
+            vs = line.chomp.split(sep, -1) 
+            key, *p = vs
+            keys[i] = key
+            parts[i] = p
+          end
+          sizes[i] ||= parts[i].length-1 unless parts[i].nil?
         end
+
         last_min = nil
         while lines.compact.any?
           min = keys.compact.sort.first
@@ -125,38 +112,62 @@ module TSV
           keys.each_with_index do |key,i|
             case key
             when min
-              str << [parts[i] * sep]
-
-              line = lines[i] = begin
-                                  streams[i].gets
-                                rescue
-                                  Log.exception $!
-                                  nil
-                                end
-              if line.nil?
-                stream = streams[i]
-                keys[i] = nil
-                parts[i] = nil
-              else
-                k, *p = line.strip.split(sep, -1)
-                keys[i] = k
-                parts[i] = p
+              str << parts[i] * sep
+
+              begin
+                line = lines[i] = begin
+                                    streams[i].gets
+                                  rescue
+                                    Log.exception $!
+                                    nil
+                                  end
+                if line.nil?
+                  stream = streams[i]
+                  keys[i] = nil
+                  parts[i] = nil
+                else
+                  k, *p = line.chomp.split(sep, -1)
+                  raise TryAgain if k == keys[i]
+                  keys[i] = k
+                  parts[i] = p.collect{|e| e.nil? ? "" : e }
+                end
+              rescue TryAgain
+                Log.warn "Skipping repeated key in stream #{i}: #{keys[i]}"
+                retry
               end
             else
-              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+              if sizes[i] > 0
+                p = sep * (sizes[i]-1)
+                str << p
+              end
             end
           end
 
-          sin.puts [min, str*sep] * sep
+          values = str.inject(nil) do |acc,part| 
+            if acc.nil?
+              acc = part.dup
+            else
+              acc << sep << part
+            end
+            acc
+          end
+          text = [min, values] * sep
+          sin.puts text
         end
 
         streams.each do |stream|
           stream.join if stream.respond_to? :join
         end
+      rescue Aborted
+        Log.error "Aborted pasting streams #{streams.inspect}: #{$!.message}"
+        streams.each do |stream|
+          stream.abort if stream.respond_to? :abort
+        end
+        raise $!
       rescue Exception
         Log.error "Exception pasting streams #{streams.inspect}: #{$!.message}"
         streams.each do |stream|
-          stream.abort
+          stream.abort if stream.respond_to? :abort
         end
         raise $!
       end

data/lib/rbbt/tsv/util.rb

@@ -142,11 +142,16 @@ module TSV
 
 
   
-  def self.header_lines(key_field, fields, entry_hash = {})
-    sep = (Hash === entry_hash and entry_hash[:sep]) ? entry_hash[:sep] : "\t"
+  def self.header_lines(key_field, fields, entry_hash = nil)
+    if Hash === entry_hash 
+      sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
+      preamble = entry_hash[:preamble]
+    end
+
+    preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
 
     str = "" 
-    str << "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if entry_hash and entry_hash.any?
+    str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
       str << "#" << key_field << sep << fields * sep << "\n"
     end