scout-gear 7.3.0 → 8.1.0

data/lib/scout/tsv/stream.rb

@@ -0,0 +1,204 @@
+module TSV
+  def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, field_prefix: nil)
+
+    streams = streams.collect do |stream|
+      case stream
+      when(defined? Step and Step)
+        stream.stream
+      when Path
+        stream.open
+      when TSV::Dumper
+        stream.stream
+      else
+        stream
+      end
+    end.compact
+
+    num_streams = streams.length
+
+    streams = streams.collect do |stream|
+      Open.sort_stream(stream, memory: sort_memory)
+    end if sort
+
+    begin
+
+      lines         =[]
+      fields        =[]
+      sizes         =[]
+      key_fields    =[]
+      input_options =[]
+      empty         =[]
+      preambles     =[]
+      parser_types  =[]
+
+      type ||= :double
+
+      streams = streams.collect do |stream|
+
+        parser = TSV::Parser.new stream, type: type
+
+        sfields = parser.fields
+
+        if field_prefix
+          index = streams.index stream
+          prefix = field_prefix[index]
+
+          sfields = sfields.collect{|f|[prefix, f]* ":"}
+        end
+
+        first_line = parser.first_line
+        first_line = nil if first_line == ""
+
+        lines         << first_line
+        key_fields    << parser.key_field
+        fields        << sfields
+        sizes         << sfields.length if sfields
+        input_options << parser.options
+        preambles     << parser.preamble      if preamble and not parser.preamble.empty?
+        parser_types  << parser.type
+
+        empty         << stream               if parser.first_line.nil? || parser.first_line.empty?
+
+        stream
+      end
+
+
+      all_fields = fields.dup
+
+      key_field = key_fields.compact.first
+
+      if same_fields
+        fields = fields.first
+      else
+        fields = fields.compact.flatten
+      end
+
+      options = input_options.first 
+      type ||= options[:type]
+      type ||= :list if type == :single
+      type ||= :double if type == :flat
+
+      preamble_txt = case preamble
+                     when TrueClass
+                       preambles * "\n"
+                     when String
+                       if preamble[0]== '+'
+                         preambles * "\n" + "\n" + preamble[1..-1]
+                       else
+                         preamble
+                       end
+                     else
+                       nil
+                     end
+
+      empty_pos = empty.collect{|stream| streams.index stream}
+
+      keys =[]
+      parts =[]
+      lines.each_with_index do |line,i|
+        if line.nil? || line.empty?
+          keys[i]= nil
+          parts[i]= nil
+        else
+          vs = line.chomp.split(sep, -1)
+          key, *p = vs
+          keys[i]= key
+          parts[i]= p
+        end
+        sizes[i]||= parts[i].length-1 unless parts[i].nil?
+      end
+      done_streams =[]
+
+      dumper = TSV::Dumper.new key_field: key_field, fields: fields, type: type
+      dumper.init 
+
+      t = Thread.new do
+        Thread.report_on_exception = false
+        Thread.current["name"] = "Paste streams"
+
+        last_min = nil
+        while lines.reject{|line| line.nil?}.any?
+          min = keys.compact.sort.first
+          break if min.nil?
+          new_values =[]
+
+          skip = all_match && keys.uniq !=[min]
+
+          keys.each_with_index do |key,i|
+            case key
+            when min
+              new_values << parts[i]
+
+              begin
+                line = lines[i]= begin
+                                   streams[i].gets
+                               rescue
+                                 Log.exception $!
+                                 nil
+                               end
+              if line.nil?
+                keys[i]= nil
+                parts[i]= nil
+              else
+                k, *p = line.chomp.split(sep, -1)
+                raise TryAgain if k == keys[i]
+                keys[i]= k
+                parts[i]= p.collect{|e| e.nil? ? "" : e}
+              end
+            rescue TryAgain
+              keys[i]= nil
+              parts[i]= nil
+              Log.debug "Skipping repeated key in stream #{i}: #{keys[i]}"
+              retry
+            end
+          else
+            p = [nil] * sizes[i]
+            new_values << p
+          end
+        end
+
+        next if skip
+
+        if same_fields
+          new_values_same = []
+          new_values.each do |list|
+            list.each_with_index do |l,i|
+              new_values_same[i] ||= []
+              new_values_same[i] << l
+            end
+          end
+          new_values = new_values_same
+        else
+          new_values = new_values.inject([]){|acc,l| acc.concat l }
+        end
+
+        dumper.add min, new_values
+      end
+
+      dumper.close
+
+      streams.each do |stream|
+        stream.close if stream.respond_to?(:close) && ! stream.closed?
+        stream.join if stream.respond_to? :join
+      end
+      end
+    rescue Aborted
+      Log.error "Aborted pasting streams #{streams.inspect}: #{$!.message}"
+      streams.each do |stream|
+        stream.abort if stream.respond_to? :abort
+      end
+      raise $!
+    rescue Exception
+      Log.error "Exception pasting streams #{streams.inspect}: #{$!.message}"
+      streams.each do |stream|
+        stream.abort if stream.respond_to? :abort
+      end
+      raise $!
+    end
+
+    Thread.pass until t["name"]
+
+    ConcurrentStream.setup(dumper.stream, threads: [t])
+  end
+
+end

data/lib/scout/tsv/transformer.rb

@@ -0,0 +1,152 @@
+module TSV
+  class Transformer
+    attr_accessor :unnamed, :parser, :dumper
+
+    def initialize(parser, dumper = nil, unnamed: false)
+      if TSV::Parser === parser
+        @parser = parser
+      elsif TSV === parser
+        @parser = parser
+      else
+        @parser = TSV::Parser.new parser
+      end
+      @unnamed = unnamed
+      if dumper.nil?
+        @dumper = TSV::Dumper.new(@parser)
+        @dumper.sep = "\t"
+      else
+        @dumper = dumper
+      end
+    end
+
+    def key_field=(key_field)
+      @dumper.key_field = key_field
+    end
+
+    def fields=(fields)
+      @dumper.fields = fields
+    end
+
+    def type=(type)
+      @dumper.type = type
+    end
+
+    def type
+      @dumper.type
+    end
+
+    def sep=(sep)
+      @dumper.sep = sep
+    end
+
+    def include?(*args)
+      false
+    end
+
+    def key_field
+      @dumper.key_field
+    end
+
+    def fields
+      @dumper.fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
+
+    def options
+      @dumper.options
+    end
+
+    def identify_field(name)
+      TSV.identify_field key_field, fields, name
+    end
+
+    def traverse(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+      end
+    end
+
+    def each(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+        [k, v]
+      end
+    end
+
+    def with_unnamed
+      begin
+        old_unnamed = @unnamed
+        @unnamed = true
+        yield
+      ensure
+        @unnamed = old_unnamed
+      end
+    end
+
+    def []=(key, value)
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      @dumper.add key, value
+    end
+
+    def stream
+      @dumper.stream
+    end
+
+    def tsv(*args)
+      TSV === @dumper ? @dumper : TSV.open(stream, *args)
+    end
+  end
+
+  def to_list
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :list
+    transformer.traverse do |k,v|
+      case self.type
+      when :single
+        [k, [v]]
+      when :double
+        [k, v.collect{|v| v.first }]
+      when :flat
+        [k, v.slice(0,1)]
+      end
+    end
+    res
+  end
+
+  def to_single
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :single
+    transformer.traverse do |k,v|
+      v = v.first while Array === v
+      [k, v]
+    end
+    res
+  end
+
+  def to_flat
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :flat
+    transformer.traverse do |k,v|
+      v = Array === v ? v.flatten : [v]
+      [k, v]
+    end
+    res
+  end
+end
+

data/lib/scout/tsv/traverse.rb

@@ -1,22 +1,14 @@
 require_relative 'parser'
 module TSV
-  def self.identify_field(key_field, fields, name)
-    return :key if name == :key || key_field.start_with?(name.to_s)
-    name.collect!{|n| key_field == n ? :key : n } if Array === name
-    NamedArray.identify_name(fields, name)
-  end
-
-  def identify_field(name)
-    TSV.identify_field(@key_field, @fields, name)
-  end
-
-  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, &block)
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
     key_field = key_field_pos if key_field.nil?
-    fields = fields_pos if fields.nil?
+    fields = fields_pos.dup if fields.nil?
     type = @type if type.nil?
     key_pos = self.identify_field(key_field)
+    fields = self.all_fields if fields == :all
     fields = [fields] unless fields.nil? || Array === fields
-    positions = fields.nil? ? nil : self.identify_field(fields)
+    positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
+
 
     if key_pos == :key
       key_name = @key_field
@@ -33,7 +25,7 @@ module TSV
       field_names = @fields
     elsif positions.nil? && key_pos != :key
       field_names = @fields.dup
-      field_names.delete_at key_pos
+      field_names.delete_at key_pos unless fields == :all
     elsif positions.include?(:key)
       field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
     else
@@ -43,98 +35,110 @@ module TSV
     key_index = positions.index :key if positions
     positions.delete :key if positions
 
-    each do |key,values|
-      values = [values] if @type == :single
-      if positions.nil?
-        if key_pos != :key
-          values = values.dup
-          key = values.delete_at(key_pos)
-        end
-      else 
-        orig_key = key
-        key = values[key_pos] if key_pos != :key 
+    log_message = "Traverse #{Log.fingerprint self}"
+    Log.debug log_message
+    bar = log_message if TrueClass === bar
 
-        values = values.values_at(*positions)
-        if key_index
-          if @type == :double
-            values.insert key_index, [orig_key]
-          else
-            values.insert key_index, orig_key
+    Log::ProgressBar.with_obj_bar(self, bar) do |bar|
+      with_unnamed unnamed do
+        each do |key,values|
+          bar.tick if bar
+          values = [values] if @type == :single
+          if positions.nil?
+            if key_pos != :key
+              values = values.dup
+              key = values.delete_at(key_pos)
+            end
+          else 
+            orig_key = key
+            key = values[key_pos] if key_pos != :key 
+
+            values = values.values_at(*positions)
+            if key_index
+              if @type == :double
+                values.insert key_index, [orig_key]
+              else
+                values.insert key_index, orig_key
+              end
+            end
           end
-        end
-      end
 
-      if Array === key 
-        if @type == :double && one2one
-          if one2one == :fill
-            key.each_with_index do |key_i,i|
-              if type == :double
-                v_i = values.collect{|v| [v[i] || v.first] }
+          values = TSV.cast_value(values, cast) if cast
+
+          if Array === key 
+            if @type == :double && one2one
+              if one2one == :strict
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i]] }
+                  else
+                    v_i = values.collect{|v| v[i] }
+                  end
+                  yield key_i, v_i
+                end
               else
-                v_i = values.collect{|v| v[i] || v.first }
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i] || v.first] }
+                  else
+                    v_i = values.collect{|v| v[i] || v.first }
+                  end
+                  yield key_i, v_i, @fields
+                end
+              end
+            else
+              key.each_with_index do |key_i, i|
+                if type == :double
+                  yield key_i, values
+                elsif type == :list
+                  yield key_i, values.collect{|v| v[i] }
+                elsif type == :flat
+                  yield key_i, values.flatten
+                elsif type == :single
+                  yield key_i, values.first
+                end
               end
-              yield key_i, v_i
             end
           else
-            key.each_with_index do |key_i,i|
-              if type == :double
-                v_i = values.collect{|v| [v[i]] }
+            if type == @type
+              if type == :single
+                yield key, values.first
               else
-                v_i = values.collect{|v| v[i] }
+                yield key, values
+              end
+            else
+              case [type, @type]
+              when [:double, :list]
+                yield key, values.collect{|v| [v] }
+              when [:double, :flat]
+                yield key, [values]
+              when [:double, :single]
+                yield key, [values]
+              when [:list, :double]
+                yield key, values.collect{|v| v.first }
+              when [:list, :flat]
+                yield key, [values.first]
+              when [:list, :single]
+                yield key, values
+              when [:flat, :double]
+                yield key, values.flatten
+              when [:flat, :list]
+                yield key, values.flatten
+              when [:flat, :single]
+                yield key, values
+              when [:single, :double]
+                yield key, values.flatten.first
+              when [:single, :list]
+                yield key, values.first
+              when [:single, :flat]
+                yield key, values.first
               end
-              yield key_i, v_i
-            end
-          end
-        else
-          key.each_with_index do |key_i, i|
-            if type == :double
-              yield key_i, values
-            elsif type == :list
-              yield key_i, values.collect{|v| v[i] }
-            elsif type == :flat
-              yield key_i, values.flatten
-            elsif type == :single
-              yield key_i, values.first
             end
           end
         end
-      else
-        if type == @type
-          if type == :single
-            yield key, values.first
-          else
-            yield key, values
-          end
-        else
-          case [type, @type]
-          when [:double, :list]
-            yield key, values.collect{|v| [v] }
-          when [:double, :flat]
-            yield key, [values]
-          when [:double, :single]
-            yield key, [values]
-          when [:list, :double]
-            yield key, values.collect{|v| v.first }
-          when [:list, :flat]
-            yield key, [values.first]
-          when [:list, :single]
-            yield key, values
-          when [:flat, :double]
-            yield key, values.flatten
-          when [:flat, :list]
-            yield key, values.flatten
-          when [:flat, :single]
-            yield key, values
-          when [:single, :double]
-            yield key, values.flatten.first
-          when [:single, :list]
-            yield key, values.first
-          when [:single, :flat]
-            yield key, values.first
-          end
-        end
       end
     end
+    
 
     [key_name, field_names]
   end

data/lib/scout/tsv/util/filter.rb

@@ -299,5 +299,14 @@ module TSV
       FileUtils.rm f
     end
   end
+
+  def with_filters(filters, &block)
+    filter
+    begin
+      filters.each{|field,value| add_filter field, value }
+    ensure
+      reset_filters
+    end
+  end
 end
 

data/lib/scout/tsv/util/reorder.rb

@@ -0,0 +1,81 @@
+require 'matrix'
+
+module TSV
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs) 
+    res = self.annotate({})
+    res.type = kwargs[:type] if kwargs.include?(:type)
+    kwargs[:one2one] = one2one
+    key_field_name, field_names = traverse key_field, fields, **kwargs do |k,v|
+      if @type == :double && merge && res.include?(k)
+        current = res[k]
+        if merge == :concat
+          v.each_with_index do |new,i|
+            next if new.empty?
+            current[i].concat(new)
+          end
+        else
+          merged = []
+          v.each_with_index do |new,i|
+            next if new.empty?
+            merged[i] = current[i] + new
+          end
+          res[k] = merged
+        end
+      else
+        res[k] = v
+      end
+    end
+    res.key_field = key_field_name
+    res.fields = field_names
+    res
+  end
+
+  def slice(fields, **kwargs)
+    reorder :key, fields, **kwargs
+  end
+
+  def column(field, **kwargs)
+    new_type = case type
+               when :double, :flat
+                 :flat
+               else
+                 :single
+               end
+
+    kwargs[:type] = new_type
+    slice(field, **kwargs)
+  end
+
+  def transpose_list(key_field="Unkown ID")
+    new_fields = keys.dup
+    new = self.annotate({})
+    TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    m = Matrix.rows values 
+    new_rows = m.transpose.to_a
+
+    fields.zip(new_rows) do |key,row|
+      new[key] = row
+    end
+
+    new
+  end
+
+  def transpose_double(key_field = "Unkown ID")
+    sep = "-!SEP--#{rand 10000}!-"
+    tmp = self.to_list{|v| v * sep}
+    new = tmp.transpose_list(key_field)
+    new.to_double{|v| v.split(sep)}
+  end
+
+  def transpose(key_field = "Unkown ID")
+    case type
+    when :single, :flat
+      self.to_list.transpose_list key_field
+    when :list
+      transpose_list key_field
+    when :double
+      transpose_double key_field
+    end
+  end
+end