scout-gear 10.4.0 → 10.6.1

data/lib/scout/tsv/parser.rb

@@ -51,7 +51,7 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kwargs, &block)
     begin
       bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
       bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
@@ -59,6 +59,32 @@ module TSV
 
       source_type = type if source_type.nil?
 
+      type_swap_key = [source_type.to_s, type.to_s] * "_"
+
+      same_type = source_type.to_s == type.to_s
+
+      if data && data.respond_to?(:load_stream) && 
+          data.serializer.to_s.include?("String") &&
+          same_type && 
+          ! (head || kwargs[:cast] || kwargs[:positions] || (kwargs[:key] && kwargs[:key] != 0) || Proc === fix ) &&
+          (kwargs[:sep].nil? || kwargs[:sep] == "\t")
+
+
+        Log.debug "Loading #{Log.fingerprint stream} directly into #{Log.fingerprint data}"
+        if first_line
+          full_stream = Open.open_pipe do |sin|
+            sin.puts first_line
+            Open.consume_stream(stream, false, sin)
+          end
+          data.load_stream(full_stream)
+        else
+          data.load_stream(stream)
+        end
+
+        return data
+      end
+
+
       data = {} if data.nil?
       merge = false if type != :double && type != :flat
       line = first_line || stream.gets
@@ -77,7 +103,7 @@ module TSV
             next
           end
 
-          key, items = parse_line(line, type: source_type, field_names: field_names, **kargs)
+          key, items = parse_line(line, type: source_type, field_names: field_names, **kwargs)
 
           next if key.nil?
 
@@ -101,38 +127,38 @@ module TSV
             end
 
             these_items = 
-              case [source_type, type]
-              when [:single, :single]
+              case type_swap_key
+              when "single_single"
                 these_items
-              when [:list, :single]
+              when "list_single"
                 these_items.first
-              when [:flat, :single]
+              when "flat_single"
                 these_items.first
-              when [:double, :single]
+              when "double_single"
                 these_items.first.first
-              when [:single, :list]
+              when "single_list"
                 [these_items]
-              when [:list, :list]
+              when "list_list"
                 these_items
-              when [:flat, :list]
+              when "flat_list"
                 these_items
-              when [:double, :list]
+              when "double_list"
                 these_items.collect{|l| l.first }
-              when [:single, :flat]
+              when "single_flat"
                 [these_items]
-              when [:list, :flat]
+              when "list_flat"
                 these_items
-              when [:flat, :flat]
+              when "flat_flat"
                 these_items
-              when [:double, :flat]
+              when "double_flat"
                 these_items.flatten
-              when [:single, :double]
+              when "single_double"
                 [[these_items]]
-              when [:list, :double]
-                these_items.collect{|l| [l] }
-              when [:flat, :double]
+              when "list_double"
+                these_items.collect{|l| l.nil? ? [] : [l] }
+              when "flat_double"
                 [these_items]
-              when [:double, :double]
+              when "double_double"
                 these_items
               end
 
@@ -143,6 +169,7 @@ module TSV
             end
 
             if ! merge || ! data.include?(key)
+              these_items = these_items.collect{|i| i.empty? ? [nil] : i } if type == :double && one2one
               data[key] = these_items
             elsif type == :double
               current = data[key]
@@ -169,6 +196,7 @@ module TSV
             end
           end
         rescue Exception
+          raise stream.stream_exception if stream.respond_to?(:stream_exception) && stream.stream_exception
           stream.abort($!) if stream.respond_to?(:abort)
           raise $!
         ensure
@@ -182,7 +210,7 @@ module TSV
       end
       data
     ensure
-      if stream.stream_exception
+      if stream.respond_to?(:stream_exception) && stream.stream_exception
         bar.remove(stream.stream_exception)
       else
         bar.remove
@@ -200,62 +228,74 @@ module TSV
   end
 
   def self.parse_header(stream, fix: true, header_hash: '#', sep: "\t")
+    sep = "\t" if sep.nil?
     if (Path === stream) || ((String === stream) && Path.is_filename?(stream))
       Open.open(stream) do |f|
         return parse_header(f, fix: fix, header_hash: header_hash, sep: sep)
       end
     end
-    raise "Closed stream" if IO === stream && stream.closed?
+
+    if IO === stream && stream.closed?
+      stream.join if stream.respond_to?(:join)
+      raise "Closed stream" 
+    end
 
     opts = {}
     preamble = []
 
     # Get line
 
-    #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
-    line = stream.gets
-    return {} if line.nil?
-    line = Misc.fixutf8 line.chomp if fix
-
-    # Process options line
-    if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
-      opts = IndiferentHash.string2hash m.captures.first.chomp
+    begin
+      #Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
       line = stream.gets
-      if line && fix
-        if Proc === fix
-          line = fix.call line
-        else
-          line = Misc.fixutf8 line.chomp if line && fix
+      return {} if line.nil?
+      line = Misc.fixutf8 line.chomp if fix
+
+      # Process options line
+      if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
+        opts = IndiferentHash.string2hash m.captures.first.chomp
+        line = stream.gets
+        if line && fix
+          if Proc === fix
+            line = fix.call line
+          else
+            line = Misc.fixutf8 line.chomp if line && fix
+          end
         end
       end
-    end
-
-    # Determine separator
-    sep = opts[:sep] if opts[:sep]
 
-    # Process fields line
-    preamble << line if line
-    while line && (TrueClass === header_hash || (String === header_hash && line.start_with?(header_hash)))
-      fields = line.split(sep, -1)
-      key_field = fields.shift
-      key_field = key_field.sub(header_hash, '') if String === header_hash && ! header_hash.empty?
+      # Determine separator
+      sep = opts[:sep] if opts[:sep]
 
-      line = (header_hash != "" ?  stream.gets : nil)
-      line = Misc.fixutf8 line.chomp if line
+      # Process fields line
       preamble << line if line
-      break if TrueClass === header_hash || header_hash == ""
-    end
+      while line && (TrueClass === header_hash || (String === header_hash && line.start_with?(header_hash)))
+        fields = line.split(sep, -1)
+        key_field = fields.shift
+        key_field = key_field.sub(header_hash, '') if String === header_hash && ! header_hash.empty?
+
+        line = (header_hash != "" ?  stream.gets : nil)
+        line = Misc.fixutf8 line.chomp if line
+        preamble << line if line
+        break if TrueClass === header_hash || header_hash == ""
+      end
 
-    preamble = preamble[0..-3] * "\n"
+      preamble = preamble[0..-3] * "\n"
 
-    line ||= stream.gets
+      line ||= stream.gets
 
-    first_line = line
+      first_line = line
 
-    opts[:type] = opts[:type].to_sym if opts[:type]
-    opts[:cast] = opts[:cast].to_sym if opts[:cast]
+      opts[:type] = opts[:type].to_sym if opts[:type]
+      opts[:cast] = opts[:cast].to_sym if opts[:cast]
 
-    NamedArray.setup([opts, key_field, fields, first_line, preamble], %w(options key_field fields first_line preamble))
+      all_fields = [key_field] + fields if key_field && fields
+      NamedArray.setup([opts, key_field, fields, first_line, preamble, all_fields], %w(options key_field fields first_line preamble all_fields))
+    rescue Exception
+      raise stream.stream_exception if stream.respond_to?(:stream_exception) && stream.stream_exception
+      stream.abort($!) if stream.respond_to?(:abort)
+      raise $!
+    end
   end
 
   KEY_PARAMETERS = begin
@@ -267,7 +307,7 @@ module TSV
                    end
 
   class Parser
-    attr_accessor :stream, :options, :key_field, :fields, :type, :first_line, :preamble
+    attr_accessor :stream, :source_options, :key_field, :fields, :type, :first_line, :preamble
     def initialize(file, fix: true, header_hash: "#", sep: "\t", type: :double)
       if IO === file
         @stream = file
@@ -275,11 +315,15 @@ module TSV
         @stream = Open.open(file)
       end
       @fix = fix
-      @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
-      @options[:filename] = file if Path.is_filename?(file)
-      @options[:sep] = sep if @options[:sep].nil?
-      @options.merge!(:key_field => @key_field, :fields => @fields)
-      @type = @options[:type] || type
+      @source_options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
+      @source_options[:filename] = file if Path.is_filename?(file)
+      @source_options[:sep] = sep if @source_options[:sep].nil?
+      @source_options.merge!(:key_field => @key_field, :fields => @fields)
+      @type = @source_options[:type] || type
+    end
+
+    def options
+      IndiferentHash.add_defaults @source_options.dup, type: type, key_field: key_field, fields: fields
     end
 
     def all_fields
@@ -288,11 +332,11 @@ module TSV
     end
 
     def key_field=(key_field)
-      @options[:key_field] = @key_field = key_field
+      @source_options[:key_field] = @key_field = key_field
     end
     
     def fields=(fields)
-      @options[:fields] = @fields = fields
+      @source_options[:fields] = @fields = fields
     end
 
     def identify_field(name)
@@ -300,7 +344,7 @@ module TSV
     end
 
     def traverse(key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
-      kwargs[:type] ||=  self.options[:type] ||= @type
+      kwargs[:type] ||=  self.source_options[:type] ||= @type
       kwargs[:type] = kwargs[:type].to_sym if kwargs[:type]
 
       if fields
@@ -308,6 +352,7 @@ module TSV
           all_field_names ||= [@key_field] + @fields
           fields = all_field_names if fields == :all
           positions = NamedArray.identify_name(all_field_names, fields)
+          raise "Not all fields (#{Log.fingerprint fields}) identified in #{Log.fingerprint all_field_names}" if positions.include?(nil)
           kwargs[:positions] = positions
           field_names = all_field_names.values_at *positions
         elsif fields.reject{|f| Numeric === f}.empty?
@@ -327,7 +372,7 @@ module TSV
           all_field_names ||= [@key_field] + @fields
           key = NamedArray.identify_name(all_field_names, key_field)
           kwargs[:key] = key == :key ? 0 : key
-          key_field_name = key === :key ? @key_field : all_field_names[key]
+          key_field_name = (key.nil? || key == :key) ? @key_field : all_field_names[key]
           if fields.nil?
             field_names = all_field_names - [key_field_name]
           end
@@ -343,21 +388,28 @@ module TSV
         field_names = field_names.slice(0,1)
       end
 
-      @options.each do |option,value|
+      @source_options.each do |option,value|
         option = option.to_sym
         next unless KEY_PARAMETERS.include? option
         kwargs[option] = value unless kwargs.include?(option)
       end
 
-      kwargs[:source_type] = @options[:type]
+      kwargs[:source_type] = @source_options[:type]
       kwargs[:data] = false if kwargs[:data].nil?
 
-      data = TSV.parse_stream(@stream, first_line: @first_line, fix: @fix, field_names: @fields, **kwargs, &block)
+      if kwargs[:tsv_grep]
+        data = with_stream do |stream|
+          grep_stream = Open.grep(stream, kwargs.delete(:tsv_grep), kwargs.delete(:tsv_invert_grep))
+          TSV.parse_stream(grep_stream, first_line: nil, fix: @fix, field_names: @fields, **kwargs, &block)
+        end
+      else
+        data = TSV.parse_stream(@stream, first_line: @first_line, fix: @fix, field_names: @fields, **kwargs, &block)
+      end
 
       if data
         TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
       else
-        [self.key_field, self.fields]
+        [key_field || self.key_field, fields || self.fields]
       end
     end
 
@@ -372,6 +424,14 @@ module TSV
     def inspect
       fingerprint
     end
+
+    def with_stream
+      sout = Open.open_pipe do |sin|
+        sin.puts @first_line
+        Open.consume_stream(@stream, false, sin)
+      end
+      yield sout
+    end
   end
 
   def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
@@ -379,40 +439,43 @@ module TSV
 
     cast = kwargs[:cast]
     cast = parser.options[:cast] if cast.nil?
+    identifiers = kwargs.delete(:identifiers)
     type = kwargs[:type] ||=  parser.options[:type] ||= :double
+
     if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
       TSV.setup(data, type: type)
       data.extend TSVAdapter
-      if serializer
-        data.serializer = serializer
-      elsif cast
-        data.serializer = 
-          case [cast, type]
-          when [:to_i, :single]
-            :integer
-          when [:to_i, :list], [:to_i, :flat]
-            :integer_array
-          when [:to_f, :single]
-            :float
-          when [:to_f, :list], [:to_f, :flat]
-            :float_array
-          else
-            type
-          end
-      else
-        data.serializer = type
-      end
+      serializer ||= if cast
+                       case [cast, type]
+                       when [:to_i, :single]
+                         :integer
+                       when [:to_i, :list], [:to_i, :flat]
+                         :integer_array
+                       when [:to_f, :single]
+                         :float
+                       when [:to_f, :list], [:to_f, :flat]
+                         :float_array
+                       when [:to_f, :double], [:to_i, :double]
+                         :marshal
+                       else
+                         type
+                       end
+                     else
+                       type
+                     end
+      data.serializer = TSVAdapter::SERIALIZER_ALIAS[serializer] || serializer
     end
 
     kwargs[:data] = {} if kwargs[:data].nil?
 
     data = parser.traverse **kwargs, &block
     data.type = type
+    data.cast = cast
     data.filename = filename || parser.options[:filename]
     data.namespace = namespace || parser.options[:namespace]
-    data.identifiers = parser.options[:identifiers]
+    data.identifiers = identifiers
     data.unnamed = unnamed
-    data.save_extension_attr_hash if data.respond_to?(:save_extension_attr_hash)
+    data.save_annotation_hash if data.respond_to?(:save_annotation_hash)
     data
   end
 end

data/lib/scout/tsv/path.rb

@@ -4,9 +4,14 @@ module Path
     TSV.open(found, *args, **kwargs, &block)
   end
 
+  def tsv_options(options = {})
+    self.open do |stream|
+      TSV::Parser.new(stream, **options).options
+    end
+  end
+
   def index(*args, **kwargs, &block)
-    found = self.find
-    found = self.set_extension('tsv').find unless found.exists?
+    found = produce_and_find('tsv')
     TSV.index(found, *args, **kwargs, &block)
   end
 end

data/lib/scout/tsv/stream.rb

@@ -9,6 +9,8 @@ module TSV
         stream.open
       when TSV::Dumper
         stream.stream
+      when TSV
+        stream.dumper_stream
       else
         stream
       end
@@ -35,7 +37,7 @@ module TSV
 
       streams = streams.collect do |stream|
 
-        parser = TSV::Parser.new stream, type: type
+        parser = TSV::Parser.new stream, type: type, sep: sep
 
         sfields = parser.fields
 
@@ -105,12 +107,13 @@ module TSV
           keys[i]= key
           parts[i]= p
         end
-        sizes[i]||= parts[i].length-1 unless parts[i].nil?
+        sizes[i] ||= parts[i].length unless parts[i].nil?
       end
       done_streams =[]
 
+      fields = nil if fields && fields.empty?
       dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type
-      dumper.init 
+      dumper.init(preamble: preamble_txt || !!key_field)
 
       t = Thread.new do
         Thread.report_on_exception = false
@@ -141,14 +144,19 @@ module TSV
                 parts[i]= nil
               else
                 k, *p = line.chomp.split(sep, -1)
-                raise TryAgain if k == keys[i]
+                p = p.collect{|e| e.nil? ? "" : e }
+
+                if k == keys[i]
+                  new_values = NamedArray.zip_fields(new_values).zip(p).collect{|p| [p.flatten * "|"] }
+                  raise TryAgain 
+                end
                 keys[i]= k
-                parts[i]= p.collect{|e| e.nil? ? "" : e}
+                parts[i]= p
               end
             rescue TryAgain
               keys[i]= nil
               parts[i]= nil
-              Log.debug "Skipping repeated key in stream #{i}: #{keys[i]}"
+              Log.debug "Skipping repeated key in stream #{i}: #{key} - #{min}"
               retry
             end
           else
@@ -201,4 +209,38 @@ module TSV
     ConcurrentStream.setup(dumper.stream, threads: [t])
   end
 
+  def self.concat_streams(streams)
+
+    streams = streams.collect do |stream|
+      case stream
+      when(defined? Step and Step)
+        stream.stream
+      when Path
+        stream.open
+      when TSV::Dumper
+        stream.stream
+      when TSV
+        stream.dumper_stream
+      else
+        stream
+      end
+    end.compact
+
+    done_streams = []
+    Open.open_pipe do |sin|
+      first_stream = streams.first
+      while line = first_stream.gets
+        sin.write line
+        break unless line[0] == "#"
+      end
+
+      while streams.any?
+        streams.each do |stream|
+          line = stream.gets
+          sin.write line unless line[0] == "#"
+        end
+        streams.delete_if{|stream| stream.eof? }
+      end
+    end
+  end
 end

data/lib/scout/tsv/transformer.rb

@@ -1,8 +1,8 @@
 module TSV
   class Transformer
-    attr_accessor :unnamed, :parser, :dumper
+    attr_accessor :unnamed, :parser, :dumper, :namespace
 
-    def initialize(parser, dumper = nil, unnamed: nil)
+    def initialize(parser, dumper = nil, unnamed: nil, namespace: nil)
       if TSV::Parser === parser
         @parser = parser
       elsif TSV === parser
@@ -68,6 +68,7 @@ module TSV
     def traverse(*args, **kwargs, &block)
       kwargs[:into] = @dumper
       kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.namespace ||= @namespace
       @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
       Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
       Open.traverse(@parser, *args, **kwargs) do |k,v|
@@ -107,7 +108,7 @@ module TSV
     end
 
     def tsv(*args)
-      TSV === @dumper ? @dumper : TSV.open(stream, *args)
+      TSV === @dumper ? @dumper : TSV.open(@dumper, *args)
     end
   end
 

data/lib/scout/tsv/traverse.rb

@@ -7,8 +7,8 @@ module TSV
     key_pos = self.identify_field(key_field)
     fields = self.all_fields if fields == :all
     fields = [fields] unless fields.nil? || Array === fields
-    positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
-
+    positions = (fields.nil? || fields == :all) ? nil : self.identify_field(fields)
+    positions = nil if fields == self.fields
 
     if key_pos == :key
       key_name = @key_field
@@ -21,8 +21,10 @@ module TSV
       end
     end
 
+    fields = positions.collect{|p| p == :key ? self.key_field : self.fields[p] } if positions
+
     if positions.nil? && key_pos == :key
-      field_names = @fields
+      field_names = @fields.dup
     elsif positions.nil? && key_pos != :key
       field_names = @fields.dup
       field_names.delete_at key_pos unless fields == :all
@@ -39,6 +41,7 @@ module TSV
     Log.debug log_message
     bar = log_message if TrueClass === bar
 
+    type_swap_tag = [type.to_s, @type.to_s] * "_"
     Log::ProgressBar.with_obj_bar(self, bar) do |bar|
       with_unnamed unnamed do
         each do |key,values|
@@ -47,13 +50,18 @@ module TSV
           if positions.nil?
             if key_pos != :key
               values = values.dup
-              key = values.delete_at(key_pos)
+              if @type == :flat
+                key = values
+              else
+                key = values.delete_at(key_pos)
+              end
             end
           else 
             orig_key = key
-            key = values[key_pos] if key_pos != :key 
+            key = @type == :flat ? values : values[key_pos] if key_pos != :key 
 
             values = values.values_at(*positions)
+            NamedArray.setup(values, fields)
             if key_index
               if @type == :double
                 values.insert key_index, [orig_key]
@@ -107,30 +115,30 @@ module TSV
                 yield key, values
               end
             else
-              case [type, @type]
-              when [:double, :list]
+              case type_swap_tag
+              when "double_list"
                 yield key, values.collect{|v| [v] }
-              when [:double, :flat]
+              when "double_flat"
                 yield key, [values]
-              when [:double, :single]
+              when "double_single"
                 yield key, [values]
-              when [:list, :double]
+              when "list_double"
                 yield key, values.collect{|v| v.first }
-              when [:list, :flat]
+              when "list_flat"
                 yield key, [values.first]
-              when [:list, :single]
+              when "list_single"
                 yield key, values
-              when [:flat, :double]
+              when "flat_double"
                 yield key, values.flatten
-              when [:flat, :list]
+              when "flat_list"
                 yield key, values.flatten
-              when [:flat, :single]
+              when "flat_single"
                 yield key, values
-              when [:single, :double]
+              when "single_double"
                 yield key, values.flatten.first
-              when [:single, :list]
+              when "single_list"
                 yield key, values.first
-              when [:single, :flat]
+              when "single_flat"
                 yield key, values.first
               end
             end

data/lib/scout/tsv/util/process.rb

@@ -70,4 +70,11 @@ module TSV
     self
   end
 
+  def remove_duplicates(pivot = 0)
+    new = self.annotate({})
+    self.through do |k,values|
+      new[k] = NamedArray.zip_fields(NamedArray.zip_fields(values).uniq)
+    end
+    new
+  end
 end