scout-gear 7.2.0 → 8.0.0

data/lib/scout/tmpfile.rb

@@ -93,19 +93,18 @@ module TmpFile
     end
   end
 
+  SLASH_REPLACE = '·'
   def self.tmp_for_file(file, tmp_options = {}, other_options = {})
-    tmp_for_file = IndiferentHash.process_options tmp_options, :file
+    tmp_for_file, prefix, key, persistence_dir = IndiferentHash.process_options tmp_options, :file, :prefix, :key, :dir
     return tmp_for_file unless tmp_for_file.nil?
 
-    prefix = IndiferentHash.process_options tmp_options, :prefix
-
     if prefix.nil?
-      perfile = file.to_s.gsub(/\//, '>') 
+      perfile = file.to_s.sub(/\.b?gz$/,'')
     else
-      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
+      perfile = prefix.to_s + ":" + file.to_s.sub(/\.b?gz$/,'')
     end
 
-    perfile.sub!(/\.b?gz$/,'')
+    perfile += "[#{ key }]" if key
 
     if other_options.include? :filters
       other_options[:filters].each do |match,value|
@@ -113,10 +112,10 @@ module TmpFile
       end
     end
 
-    persistence_dir = IndiferentHash.process_options(tmp_options, :dir) || TmpFile.tmpdir 
+    persistence_dir = TmpFile.tmpdir if persistence_dir.nil?
     Path.setup(persistence_dir) unless Path === persistence_dir
 
-    filename = perfile.gsub(/\s/,'_').gsub(/\//,'>')
+    filename = perfile.gsub(/\s/,'_').gsub('/', SLASH_REPLACE)
     clean_options = other_options.dup
     clean_options.delete :unnamed
     clean_options.delete "unnamed"

data/lib/scout/tsv/attach.rb

@@ -0,0 +1,177 @@
+module TSV
+
+  def self.match_keys(source, other, match_key: nil, other_key: nil)
+    match_key = (source.all_fields & other.all_fields).first if match_key.nil?
+
+    if match_key.nil?
+      source.all_fields.collect do |f|
+        other_key = other.identify_field(f)
+        if other_key
+          other_key = other.key_field if other_key == :key
+          match_key = f
+          break
+        end
+      end
+    end
+
+    if match_key.nil?
+      other.all_fields.collect do |f|
+        match_key = source.identify_field(f)
+        if match_key
+          other_key = f
+          break
+        end
+      end
+    end
+
+    match_key = source.key_field if match_key.nil? 
+
+    if other_key.nil?
+      other_key = other.identify_field(match_key)
+    end
+
+    other_key = other.key_field if other_key.nil?
+
+    match_key = :key if match_key == source.key_field
+    other_key = :key if other_key == other.key_field
+
+    [match_key, other_key]
+  end
+
+  def self.attach(source, other, target: nil, fields: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
+    source = TSV::Transformer.new source unless TSV === source || TSV::Parser === source
+    other = TSV.open other, persist: persist_input unless TSV === other 
+
+    fields = [fields] if String === fields
+
+    match_key, other_key = TSV.match_keys(source, other, match_key: match_key, other_key: other_key)
+
+    if TSV::Transformer === source
+      source.dumper = case target
+                      when :stream
+                        TSV::Dumper.new(source.options.merge(sep: "\t"))
+                      when nil
+                        TSV.setup({}, **source.options.dup)
+                      else
+                        target
+                      end
+    end
+
+    other.with_unnamed do
+      source.with_unnamed do
+
+        other_key_name = other_key == :key ? other.key_field : other_key
+        other_key_name = other.fields[other_key_name] if Integer === other_key
+        fields = other.all_fields - [other_key_name, source.key_field] if fields.nil?
+
+        if other_key != :key 
+          other = other.reorder other_key, fields, one2one: one2one
+        end
+
+        other_field_positions = other.identify_field(fields) 
+
+        log_message = "Attach #{Log.fingerprint fields - source.fields} to #{Log.fingerprint source} (#{[match_key, other_key] * "=~"})"
+        Log.debug log_message
+        bar = log_message if TrueClass === bar
+
+        new = fields - source.fields
+
+        source.fields = (source.fields + fields).uniq
+
+        overlaps = source.identify_field(fields)
+
+        empty_other_values = case source.type
+                             when :list
+                               [nil] * other.fields.length
+                             when :flat
+                               []
+                             when :double
+                               [[]] * other.fields.length
+                             end
+
+        insitu = TSV === source ? true : false if insitu.nil?
+
+        match_key_pos = source.identify_field(match_key)
+        source.traverse bar: bar, unnamed: true do |orig_key,current_values|
+          keys = (match_key == :key || match_key_pos == :key) ? [orig_key] : current_values[match_key_pos]
+          keys = [keys] unless Array === keys
+
+          current_values = current_values.dup unless insitu
+          keys.each do |current_key|
+            other_values = other[current_key]
+
+            if other_values.nil?
+              other_values = empty_other_values
+            elsif other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            other_values = other_values.values_at *other_field_positions
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if source.type == :list
+                current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
+              else
+                current_values[overlap] ||= []
+                current_values[overlap].concat (v - current_values[overlap])
+              end
+            end
+          end
+          source[orig_key] = current_values unless insitu
+          nil
+        end
+
+        if complete && match_key == :key
+          empty_self_values = case source.type
+                              when :list
+                                [nil] * source.fields.length
+                              when :flat
+                                []
+                              when :double
+                                [[]] * source.fields.length
+                              end
+          other.each do |other_key,other_values|
+            next if source.include?(other_key)
+            if other.type == :flat
+              other_values = [other_values]
+            elsif other.type == :list && source.type == :double
+              other_values = other_values.collect{|v| [v] }
+            elsif other.type == :double && source.type == :list
+              other_values = other_values.collect{|v| v.first }
+            end
+
+            new_values = case source.type
+                         when :list
+                           [nil] * source.fields.length
+                         when :flat
+                           []
+                         when :double
+                           source.fields.length.times.collect{ [] }
+                         end
+
+            other_values.zip(overlaps).each do |v,overlap|
+              if false && overlap == :key
+                other_key = Array === v ? v : v.first
+              elsif source.type == :list
+                new_values[overlap] = v if v[overlap].nil? || String === v[overlap] && v[overlap].empty?
+              else
+                new_values[overlap].concat v
+              end
+            end
+            source[other_key] = new_values
+          end
+        end
+      end
+    end
+
+    source
+  end
+
+  def attach(*args, **kwargs)
+    TSV.attach(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/change_id.rb

@@ -0,0 +1,40 @@
+module TSV
+  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, stream: false, keep: false, persist_identifiers: nil)
+    source = TSV::Parser.new source if String === source
+    if identifiers && source.identify_field(new_key_field, strict: true).nil?
+      identifiers = identifiers.nil? ? source.identifiers : identifiers
+      new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
+      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one)
+      return new
+    end
+
+    fields = source.fields.dup - [new_key_field]
+    fields.unshift source.key_field if keep
+    transformer = TSV::Transformer.new source
+    transformer.key_field = new_key_field
+    transformer.fields = fields
+    transformer.traverse key_field: new_key_field, fields: fields, one2one: one2one, unnamed: true do |k,v|
+      [k, v]
+    end
+
+    stream ? transformer : transformer.tsv
+  end
+
+  def change_key(*args, **kwargs)
+    TSV.change_key(self, *args, **kwargs)
+  end
+
+  def self.change_id(source, source_id, new_id, identifiers: nil, one2one: false, insitu: false)
+    source = TSV::Parser.new source if String === source
+
+    identifiers = identifiers.nil? ? source.identifiers : identifiers
+
+    new_fields = source.fields.dup
+    new_fields[new_fields.index(source_id)] = new_id
+    return source.attach(identifiers, fields: [new_id], insitu: insitu).slice(new_fields)
+  end
+
+  def change_id(*args, **kwargs)
+    TSV.change_id(self, *args, **kwargs)
+  end
+end

data/lib/scout/tsv/dumper.rb

@@ -1,38 +1,14 @@
 module TSV
   class Dumper
-    def self.header_lines(key_field, fields, entry_hash = nil)
-      if Hash === entry_hash 
-        sep = entry_hash[:sep] ? entry_hash[:sep] : "\t"
-        preamble = entry_hash[:preamble]
-        header_hash = entry_hash[:header_hash]
-      end
-
-      header_hash = "#" if header_hash.nil?
-
-      preamble = "#: " << Misc.hash2string(entry_hash.merge(:key_field => nil, :fields => nil)) << "\n" if preamble.nil? and entry_hash and entry_hash.values.compact.any?
-
-      str = "" 
-      str << preamble.strip << "\n" if preamble and not preamble.empty?
-      if fields
-        if fields.empty?
-          str << header_hash << (key_field || "ID").to_s << "\n" 
-        else
-          str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
-        end
-      end
-
-      str
-    end
-
     def self.header(options={})
-      key_field, fields, sep, header_hash, preamble = IndiferentHash.process_options options, 
-        :key_field, :fields, :sep, :header_hash, :preamble,
+      key_field, fields, sep, header_hash, preamble, unnamed = IndiferentHash.process_options options, 
+        :key_field, :fields, :sep, :header_hash, :preamble, :unnamed,
         :sep => "\t", :header_hash => "#", :preamble => true
 
-      if fields.nil? || key_field.nil?
+      if fields.nil?
         fields_str = nil
       else
-        fields_str = "#{header_hash}#{key_field}#{sep}#{fields*sep}"
+        fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
       end
 
       if preamble && options.values.compact.any?
@@ -45,31 +21,62 @@ module TSV
     end
 
 
-    attr_accessor :options
+    attr_accessor :options, :initialized, :type, :sep
     def initialize(options = {})
+      options = options.options.merge(sep: nil) if TSV::Parser === options || TSV === options
       @sep, @type = IndiferentHash.process_options options, 
         :sep, :type, 
         :sep => "\t", :type => :double
       @options = options
       @sout, @sin = Open.pipe
-      ConcurrentStream.setup(@sin, :pair => @sout)
-      ConcurrentStream.setup(@sout, :pair => @sin)
+      @initialized = false
+      @mutex = Mutex.new
+      ConcurrentStream.setup(@sin, pair: @sout)
+      ConcurrentStream.setup(@sout, pair: @sin)
     end
 
-    def init
-      header = Dumper.header(@options.merge(:type => @type, :sep => @sep))
-      @sin.puts header if header and ! header.empty?
+    def key_field
+      @options[:key_field]
+    end
+    
+    def fields
+      @options[:fields]
     end
 
-    def add(key, value)
+    def key_field=(key_field)
+      @options[:key_field] = key_field
+    end
+    
+    def fields=(fields)
+      @options[:fields] = fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
 
-      case @type
-      when :single
-        @sin.puts key + @sep + value
-      when :list, :flat
-        @sin.puts key + @sep + value * @sep
-      when :double
-        @sin.puts key + @sep + value.collect{|v| v * "|" } * @sep
+
+    def init(preamble: true)
+      header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
+      @mutex.synchronize do
+        @initialized = true
+        @sin.puts header if header and ! header.empty?
+      end
+    end
+
+    def add(key, value)
+      @mutex.synchronize do
+
+        key = key.to_s unless String === key
+        case @type
+        when :single
+          @sin.puts key + @sep + value.to_s
+        when :list, :flat
+          @sin.puts key + @sep + value * @sep
+        when :double
+          @sin.puts key + @sep + value.collect{|v| Array === v ? v * "|" : v } * @sep
+        end
       end
     end
 
@@ -85,23 +92,47 @@ module TSV
     def abort(exception=nil)
       @sin.abort(exception)
     end
+
+    def tsv(*args)
+      TSV.open(stream, *args)
+    end
+
+    def fingerprint
+      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+    end
+
+    def digest_str
+      fingerprint
+    end
+
+    def inspect
+      fingerprint
+    end
   end
 
-  def stream
-    iii self.extension_attr_hash
-    dumper = TSV::Dumper.new self.extension_attr_hash
-    dumper.init
-    Thread.new do 
-      Thread.current["name"] = "Dumper thread"
-      self.each do |k,v|
-        dumper.add k, v
+  def dumper_stream(options = {})
+    preamble = IndiferentHash.process_options options, :preamble, :preamble => true
+    dumper = TSV::Dumper.new self.extension_attr_hash.merge(options)
+    t = Thread.new do 
+      begin
+        Thread.current.report_on_exception = true
+        Thread.current["name"] = "Dumper thread"
+        dumper.init(preamble: preamble)
+        self.each do |k,v|
+          dumper.add k, v
+        end
+        dumper.close
+      rescue
+        dumper.abort($!)
       end
-      dumper.close
     end
-    dumper.stream
+    Thread.pass until t["name"]
+    s = dumper.stream
+    ConcurrentStream.setup(s, :threads => [t])
+    s
   end
 
-  def to_s
-    stream.read
+  def to_s(options = {})
+    dumper_stream(options).read
   end
 end

data/lib/scout/tsv/index.rb

@@ -1,12 +1,16 @@
 require_relative 'parser'
+require_relative 'transformer'
+require_relative 'persist/fix_width_table'
 module TSV
-  def self.index(tsv_file, target: 0, order: true, **kwargs)
-    persist, type = IndiferentHash.process_options kwargs,
-      :persist, :persist_type,
+  def self.index(tsv_file, target: 0, fields: nil, order: true, bar: nil, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => "HDB"
     kwargs.delete :type
 
-    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+    fields = :all if fields.nil?
+
+    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, :prefix => "Index", :other_options => kwargs)) do |filename|
       if filename
         index = ScoutCabinet.open(filename, true, type)
         TSV.setup(index, :type => :single)
@@ -15,35 +19,199 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      dummy_data = nil
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      bar = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}" if TrueClass === bar
+
       if order
         tmp_index = {}
-        dummy_data = Open.open(tsv_file) do |file|
-          TSV.parse file, key_field: target, type: :double, **kwargs do |k,values|
-            values.each_with_index do |list,i|
-              list.each do |e|
-                tmp_index[e] ||= []
-                tmp_index[e][i] ||= []
-                tmp_index[e][i] << k
-              end
+        include_self = fields == :all || (Array === fields) && fields.include?(target)
+        target_key_field, source_field_names = Open.traverse tsv_file, key_field: target, fields: fields, type: :double, unnamed: true, bar: bar, **kwargs do |k,values|
+          tmp_index[k] ||= [[k]] if include_self
+          values.each_with_index do |list,i|
+            i += 1 if include_self
+            list.each do |e|
+              tmp_index[e] ||= []
+              tmp_index[e][i] ||= []
+              tmp_index[e][i] << k
             end
           end
         end
         tmp_index.each do |e,list|
           index[e] = list.flatten.compact.uniq.first
         end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
+
+        tmp_index = {}
+
       else
-        dummy_data = Open.open(tsv_file) do |file|
-          TSV.parse file, key_field: target, type: :flat, **kwargs do |k,values|
-            values.each do |e|
-              index[e] = k unless index.include?(e)
-            end
+        target_key_field, source_field_names =  Open.traverse tsv_file, key_field: target, fields: fields, type: :flat, unnamed: true, bar: bar, **kwargs do |k,values|
+          values.each do |e|
+            index[e] = k unless index.include?(e)
+          end
+        end
+
+        index.key_field = source_field_names * ","
+        index.fields = [target_key_field]
+      end
+
+
+      index
+    end
+  end
+
+  def index(*args, **kwargs, &block)
+    TSV.index(self, *args, **kwargs, &block)
+  end
+
+  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :prefix => "RangeIndex", :other_options => kwargs, update: persist_update)) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field], **kwargs do |key, values|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        start_pos, end_pos = values
+        if Array === start_pos
+          start_pos.zip(end_pos).each do |s,e|
+            index_data << [key, [s.to_i, e.to_i]]
           end
+        else
+          index_data << [key, [start_pos.to_i, end_pos.to_i]]
         end
       end
-      index.key_field = dummy_data.fields * ", "
-      index.fields = [dummy_data.key_field]
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, true)
+      index.add_range index_data
+      index.read
       index
     end
   end
+
+  def range_index(*args, **kwargs, &block)
+    TSV.range_index(self, *args, **kwargs, &block)
+  end
+
+  def self.pos_index(tsv_file, pos_field = nil, key_field: :key, **kwargs)
+    persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
+      :persist, :persist_type, :persist_update, :data_persist,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, update: persist_update, :prefix => "RangeIndex", :other_options => kwargs)) do |filename|
+
+      tsv_file = TSV.open(tsv_file, persist: true) if data_persist && ! TSV === tsv_file
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [pos_field], type: :single, cast: :to_i, **kwargs do |key, pos|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        if Array === pos
+          pos.zip(end_pos).each do |p|
+            index_pos << [key, p]
+          end
+        else
+          index_data << [key, pos]
+        end
+      end
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, false)
+      index.add_point index_data
+      index.read
+      index
+    end
+  end
+
+  def pos_index(*args, **kwargs, &block)
+    TSV.pos_index(self, *args, **kwargs, &block)
+  end
+
+
+  #def range_index(start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  options = Misc.add_defaults options,
+  #    :persist => false, :persist_file => nil, :persist_update => false 
+
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
+
+  #  Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do 
+  #    max_key_size = 0
+  #    index_data = []
+  #    with_unnamed do
+  #      with_monitor :desc => "Creating Index Data", :step => 10000 do
+  #        through :key, [start_field, end_field] do |key, values|
+  #          key_size = key.length
+  #          max_key_size = key_size if key_size > max_key_size
+
+  #          start_pos, end_pos = values
+  #          if Array === start_pos
+  #            start_pos.zip(end_pos).each do |s,e|
+  #              index_data << [key, [s.to_i, e.to_i]]
+  #            end
+  #          else
+  #            index_data << [key, [start_pos.to_i, end_pos.to_i]]
+  #          end
+  #        end
+  #      end
+  #    end
+
+  #    index = FixWidthTable.get(:memory, max_key_size, true)
+  #    index.add_range index_data
+  #    index.read
+  #    index
+  #  end
+  #end
+
+  #def self.range_index(file, start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  data_options = Misc.pull_keys options, :data
+  #  filename = case
+  #             when (String === file or Path === file)
+  #               file
+  #             when file.respond_to?(:filename)
+  #               file.filename
+  #             else
+  #               file.object_id.to_s
+  #             end
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
+
+  #  filters = Misc.process_options options, :filters
+
+  #  if filters
+  #    filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
+  #  end
+
+  #  Persist.persist(filename, :fwt, persist_options) do
+  #    tsv = TSV.open(file, data_options)
+  #    if filters
+  #      tsv.filter
+  #      filters.each do |match, value|
+  #        tsv.add_filter match, value
+  #      end
+  #    end
+ 
+  #    tsv.range_index(start_field, end_field, options)
+  #  end
+  #end
 end