scout-gear 7.3.0 → 8.0.0

data/lib/scout/tsv/attach.rb

@@ -0,0 +1,177 @@
+module TSV
+
+  def self.match_keys(source, other, match_key: nil, other_key: nil)
+    match_key = (source.all_fields & other.all_fields).first if match_key.nil?
+
+    if match_key.nil?
+      source.all_fields.collect do |f|
+        other_key = other.identify_field(f)
+        if other_key
+          other_key = other.key_field if other_key == :key
+          match_key = f
+          break
+        end
+      end
+    end
+
+    if match_key.nil?
+      other.all_fields.collect do |f|
+        match_key = source.identify_field(f)
+        if match_key
+          other_key = f
+          break
+        end
+      end
+    end
+
+    match_key = source.key_field if match_key.nil? 
+
+    if other_key.nil?
+      other_key = other.identify_field(match_key)
+    end
+
+    other_key = other.key_field if other_key.nil?
+
+    match_key = :key if match_key == source.key_field
+    other_key = :key if other_key == other.key_field
+
+    [match_key, other_key]
+  end
+
+  def self.attach(source, other, target: nil, fields: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
+    source = TSV::Transformer.new source unless TSV === source || TSV::Parser === source
+    other = TSV.open other, persist: persist_input unless TSV === other 
+
+    fields = [fields] if String === fields
+
+    match_key, other_key = TSV.match_keys(source, other, match_key: match_key, other_key: other_key)
+
+    if TSV::Transformer === source
+      source.dumper = case target
+                      when :stream
+                        TSV::Dumper.new(source.options.merge(sep: "\t"))
+                      when nil
+                        TSV.setup({}, **source.options.dup)
+                      else
+                        target
+                      end
+    end
+
+    other.with_unnamed do
+      source.with_unnamed do
+
+        other_key_name = other_key == :key ? other.key_field : other_key
+        other_key_name = other.fields[other_key_name] if Integer === other_key
+        fields = other.all_fields - [other_key_name, source.key_field] if fields.nil?
+
+        if other_key != :key 
+          other = other.reorder other_key, fields, one2one: one2one
+        end
+
+        other_field_positions = other.identify_field(fields) 
+
+        log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
+        Log.debug log_message
+        bar = log_message if TrueClass === bar
+
+        new = fields - source.fields
+
+        source.fields = (source.fields + fields).uniq
+
+        overlaps = source.identify_field(fields)
+
+        empty_other_values = case source.type
+                             when :list
+                               [nil] * other.fields.length
+                             when :flat
+                               []
+                             when :double
+                               [[]] * other.fields.length
+                             end
+
+        insitu = TSV === source ? true : false if insitu.nil?
+
+        match_key_pos = source.identify_field(match_key)
+        source.traverse bar: bar, unnamed: true do |orig_key,current_values|
+          keys = (match_key == :key || match_key_pos == :key) ? [orig_key] : current_values[match_key_pos]
+          keys = [keys] unless Array === keys
+
+          current_values = current_values.dup unless insitu
+          keys.each do |current_key|
+            other_values = other[current_key]
+
+            if other_values.nil?
+              other_values = empty_other_values
+            elsif other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            other_values = other_values.values_at *other_field_positions
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if source.type == :list
+                current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
+              else
+                current_values[overlap] ||= []
+                current_values[overlap].concat (v - current_values[overlap])
+              end
+            end
+          end
+          source[orig_key] = current_values unless insitu
+          nil
+        end
+
+        if complete && match_key == :key
+          empty_self_values = case source.type
+                              when :list
+                                [nil] * source.fields.length
+                              when :flat
+                                []
+                              when :double
+                                [[]] * source.fields.length
+                              end
+          other.each do |other_key,other_values|
+            next if source.include?(other_key)
+            if other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            new_values = case source.type
+                         when :list
+                           [nil] * source.fields.length
+                         when :flat
+                           []
+                         when :double
+                           source.fields.length.times.collect{ [] }
+                         end
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if false && overlap == :key
+                other_key = Array === v ? v : v.first
+              elsif source.type == :list
+                new_values[overlap] = v if v[overlap].nil? || String === v[overlap] && v[overlap].empty?
+              else
+                new_values[overlap].concat v
+              end
+            end
+            source[other_key] = new_values
+          end
+        end
+      end
+    end
+
+    source
+  end
+
+  def attach(*args, **kwargs)
+    TSV.attach(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/change_id.rb

@@ -0,0 +1,40 @@
+module TSV
+  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, stream: false, keep: false, persist_identifiers: nil)
+    source = TSV::Parser.new source if String === source
+    if identifiers && source.identify_field(new_key_field, strict: true).nil?
+      identifiers = identifiers.nil? ? source.identifiers : identifiers
+      new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
+      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one)
+      return new
+    end
+
+    fields = source.fields.dup - [new_key_field]
+    fields.unshift source.key_field if keep
+    transformer = TSV::Transformer.new source
+    transformer.key_field = new_key_field
+    transformer.fields = fields
+    transformer.traverse key_field: new_key_field, fields: fields, one2one: one2one, unnamed: true do |k,v|
+      [k, v]
+    end
+
+    stream ? transformer : transformer.tsv
+  end
+
+  def change_key(*args, **kwargs)
+    TSV.change_key(self, *args, **kwargs)
+  end
+
+  def self.change_id(source, source_id, new_id, identifiers: nil, one2one: false, insitu: false)
+    source = TSV::Parser.new source if String === source
+
+    identifiers = identifiers.nil? ? source.identifiers : identifiers
+
+    new_fields = source.fields.dup
+    new_fields[new_fields.index(source_id)] = new_id
+    return source.attach(identifiers, fields: [new_id], insitu: insitu).slice(new_fields)
+  end
+
+  def change_id(*args, **kwargs)
+    TSV.change_id(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/dumper.rb

@@ -1,38 +1,14 @@
 module TSV
   class Dumper
-    def self.header_lines(key_field, fields, entry_hash = nil)
-      if Hash === entry_hash 
-        sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
-        preamble = entry_hash[:preamble]
-        header_hash = entry_hash[:header_hash]
-      end
-
-      header_hash = "#" if header_hash.nil?
-
-      preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
-
-      str = "" 
-      str << preamble.strip << "\n" if preamble and not preamble.empty?
-      if fields
-        if fields.empty?
-          str << header_hash << (key_field || "ID").to_s << "\n" 
-        else
-          str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
-        end
-      end
-
-      str
-    end
-
     def self.header(options={})
-      key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options, 
-        :key_field, :fields, :sep, :header_hash, :preamble,
+      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options, 
+        :key_field, :fields, :sep, :header_hash, :preamble, :unnamed,
         :sep => "\t", :header_hash => "#", :preamble => true
 
-      if fields.nil? || key_field.nil?
+      if fields.nil?
         fields_str = nil
       else
-        fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
+        fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
       end
 
       if preamble && options.values.compact.any?
@@ -45,31 +21,62 @@ module TSV
     end
 
 
-    attr_accessor :options
+    attr_accessor :options, :initialized, :type, :sep
     def initialize(options = {})
+      options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
       @sep, @type = IndiferentHash.process_options options, 
         :sep, :type, 
         :sep => "\t", :type => :double
       @options = options
       @sout, @sin = Open.pipe
+      @initialized = false
+      @mutex = Mutex.new
       ConcurrentStream.setup(@sin, pair: @sout)
       ConcurrentStream.setup(@sout, pair: @sin)
     end
 
-    def init
-      header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
-      @sin.puts header if header and ! header.empty?
+    def key_field
+      @options[:key_field]
+    end
+    
+    def fields
+      @options[:fields]
     end
 
-    def add(key, value)
+    def key_field=(key_field)
+      @options[:key_field] = key_field
+    end
+    
+    def fields=(fields)
+      @options[:fields] = fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
 
-      case @type
-      when :single
-        @sin.puts key + @sep + value
-      when :list, :flat
-        @sin.puts key + @sep + value * @sep
-      when :double
-        @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
+
+    def init(preamble: true)
+      header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
+      @mutex.synchronize do
+        @initialized = true
+        @sin.puts header if header and ! header.empty?
+      end
+    end
+
+    def add(key, value)
+      @mutex.synchronize do
+
+        key = key.to_s unless String === key
+        case @type
+        when :single
+          @sin.puts key + @sep + value.to_s
+        when :list, :flat
+          @sin.puts key + @sep + value * @sep
+        when :double
+          @sin.puts key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep
+        end
       end
     end
 
@@ -85,15 +92,32 @@ module TSV
     def abort(exception=nil)
       @sin.abort(exception)
     end
+
+    def tsv(*args)
+      TSV.open(stream, *args)
+    end
+
+    def fingerprint
+      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+    end
+
+    def digest_str
+      fingerprint
+    end
+
+    def inspect
+      fingerprint
+    end
   end
 
-  def stream
-    dumper = TSV::Dumper.new self.extension_attr_hash
-    dumper.init
+  def dumper_stream(options = {})
+    preamble = IndiferentHash.process_options options, :preamble, :preamble => true
+    dumper = TSV::Dumper.new self.extension_attr_hash.merge(options)
     t = Thread.new do 
       begin
         Thread.current.report_on_exception = true
         Thread.current["name"] = "Dumper thread"
+        dumper.init(preamble: preamble)
         self.each do |k,v|
           dumper.add k, v
         end
@@ -103,10 +127,12 @@ module TSV
       end
     end
     Thread.pass until t["name"]
-    dumper.stream
+    s = dumper.stream
+    ConcurrentStream.setup(s, :threads => [t])
+    s
   end
 
-  def to_s
-    stream.read
+  def to_s(options = {})
+    dumper_stream(options).read
   end
 end

data/lib/scout/tsv/index.rb

@@ -1,13 +1,16 @@
 require_relative 'parser'
+require_relative 'transformer'
 require_relative 'persist/fix_width_table'
 module TSV
-  def self.index(tsv_file, target: 0, fields: nil, order: true, **kwargs)
-    persist, type = IndiferentHash.process_options kwargs,
-      :persist, :persist_type,
+  def self.index(tsv_file, target: 0, fields: nil, order: true, bar: nil, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => "HDB"
     kwargs.delete :type
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+    fields = :all if fields.nil?
+
+    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, :prefix => "Index", :other_options => kwargs)) do |filename|
       if filename
         index = ScoutCabinet.open(filename, true, type)
         TSV.setup(index, :type => :single)
@@ -16,11 +19,17 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      dummy_data = TSV.setup({}, :key_field => "Key", :fields => ["Target"])
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      bar = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}" if TrueClass === bar
+
       if order
         tmp_index = {}
-        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :double, into: dummy_data, unnamed: true, **kwargs do |k,values|
+        include_self = fields == :all || (Array === fields) && fields.include?(target)
+        target_key_field, source_field_names = Open.traverse tsv_file, key_field: target, fields: fields, type: :double, unnamed: true, bar: bar, **kwargs do |k,values|
+          tmp_index[k] ||= [[k]] if include_self
           values.each_with_index do |list,i|
+            i += 1 if include_self
             list.each do |e|
               tmp_index[e] ||= []
               tmp_index[e][i] ||= []
@@ -31,16 +40,24 @@ module TSV
         tmp_index.each do |e,list|
           index[e] = list.flatten.compact.uniq.first
         end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
+
+        tmp_index = {}
+
       else
-        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :flat, into: dummy_data, unnamed: true, **kwargs do |k,values|
+        target_key_field, source_field_names =  Open.traverse tsv_file, key_field: target, fields: fields, type: :flat, unnamed: true, bar: bar, **kwargs do |k,values|
           values.each do |e|
             index[e] = k unless index.include?(e)
           end
         end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
       end
 
-      index.key_field = dummy_data.fields * ", "
-      index.fields = [dummy_data.key_field]
+
       index
     end
   end
@@ -50,16 +67,18 @@ module TSV
   end
 
   def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
-    persist, type = IndiferentHash.process_options kwargs,
-      :persist, :persist_type,
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => :fwt
     kwargs.delete :type
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :prefix => "RangeIndex", :other_options => kwargs, update: persist_update)) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
 
       max_key_size = 0
       index_data = []
-      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field] do |key, values|
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], **kwargs do |key, values|
         key_size = key.length
         max_key_size = key_size if key_size > max_key_size
 
@@ -85,6 +104,43 @@ module TSV
     TSV.range_index(self, *args, **kwargs, &block)
   end
 
+  def self.pos_index(tsv_file, pos_field = nil, key_field: :key, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, update: persist_update, :prefix => "RangeIndex", :other_options => kwargs)) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, **kwargs do |key, pos|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        if Array === pos
+          pos.zip(end_pos).each do |p|
+            index_pos << [key, p]
+          end
+        else
+          index_data << [key, pos]
+        end
+      end
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, false)
+      index.add_point index_data
+      index.read
+      index
+    end
+  end
+
+  def pos_index(*args, **kwargs, &block)
+    TSV.pos_index(self, *args, **kwargs, &block)
+  end
+
 
   #def range_index(start_field = nil, end_field = nil, options = {})
   #  start_field ||= "Start"