scout-gear 7.3.0 → 8.0.0

data/lib/scout/tsv/persist/adapter.rb

@@ -1,8 +1,9 @@
 require_relative '../../open/lock'
+require_relative '../../meta_extension'
 require_relative 'serialize'
 
 module TSVAdapter
-  attr_accessor :persistence_path, :persistence_class, :serializer, :closed, :writable
+  attr_accessor :persistence_path, :persistence_class, :closed, :writable, :serializer
 
   class << self
     attr_accessor :lock_dir
@@ -19,23 +20,15 @@ module TSVAdapter
   end
 
   def load_extension_attr_hash
-    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self[EXTENSION_ATTR_HASH_KEY]))
+    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(EXTENSION_ATTR_HASH_KEY)))
   end
 
   def save_extension_attr_hash
-    self[EXTENSION_ATTR_HASH_KEY]= EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash)
+    self.orig_set(EXTENSION_ATTR_HASH_KEY, EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash))
   end
 
   def self.extended(base)
-    if ! TSVAdapter === base
-      if (! TSVAdapter === base) && base.include?(EXTENSION_ATTR_HASH_KEY)
-        TSV.setup(base, base.load_extension_attr_hash)
-      elsif TSV === base
-        base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
-      end
-    end
-
-    base.serializer = SERIALIZER_ALIAS[base.type]
+    return if base.respond_to?(:orig_set)
 
     class << base
       alias orig_set []=
@@ -54,45 +47,22 @@ module TSVAdapter
       end
 
       def load_value(str)
+        return nil if str.nil?
         serializer.load(str)
       end
 
       def save_value(value)
         serializer.dump(value)
       end
-
-
     end
 
-    #case base.type
-    #when :single
-    #  class << base
-    #    def load_value(value)
-    #      value
-    #    end
-    #    def save_value(value)
-    #      value
-    #    end
-    #  end
-    #when :list, :flat
-    #  class << base
-    #    def load_value(value)
-    #      value.nil? ? nil : value.split("\t")
-    #    end
-    #    def save_value(value)
-    #      value * "\t"
-    #    end
-    #  end
-    #when :double
-    #  class << base
-    #    def load_value(value)
-    #      value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
-    #    end
-    #    def save_value(value)
-    #      value.collect{|v| v * "|" } * "\t"
-    #    end
-    #  end
-    #end
+    if base.include?(EXTENSION_ATTR_HASH_KEY)
+      TSV.setup(base, base.load_extension_attr_hash)
+    else
+      base.instance_variable_get(:@extension_attrs).push :serializer
+      base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
+      base.save_extension_attr_hash
+    end
   end
 
   def keys(*args)
@@ -101,9 +71,9 @@ module TSVAdapter
     end
 
     if k[0] == EXTENSION_ATTR_HASH_KEY
-      k.slice(1,k.length-1)
+      k.slice(1,k.length)
     elsif k[-1] == EXTENSION_ATTR_HASH_KEY
-      k.slice(0,k.length-2)
+      k.slice(0,k.length-1)
     else
       k - [EXTENSION_ATTR_HASH_KEY]
     end

data/lib/scout/tsv/persist/fix_width_table.rb

@@ -2,6 +2,7 @@ class FixWidthTable
 
   attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
+    filename = filename.find if Path === filename
     @filename = filename
 
     if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
@@ -312,6 +313,8 @@ class FixWidthTable
       acc.concat(new)
     end
   end
+
+  alias length size
 end
 
 Persist.save_drivers[:fwt] = proc do |file, content|

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -16,10 +16,13 @@ module ScoutCabinet
     dir = File.dirname(File.expand_path(path))
     Open.mkdir(dir) unless File.exist?(dir)
 
+    tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
     tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
     tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    database = Persist::CONNECTIONS[path] ||= tokyocabinet_class.new
+    # Hack - Ignore warning: undefining the allocator of T_DATA class
+    # TokyoCabinet::HDB_data
+    database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
     if big and not Open.exists?(path)
       database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 

data/lib/scout/tsv/persist.rb

@@ -6,6 +6,10 @@ Persist.save_drivers[:tsv] = proc do |file,content|
              content
            elsif content.respond_to?(:stream)
              content.stream
+           elsif content.respond_to?(:dumper_stream)
+             content.dumper_stream
+           else
+             content
            end
   Open.sensible_write(file, stream)
 end

data/lib/scout/tsv/transformer.rb

@@ -0,0 +1,141 @@
+module TSV
+  class Transformer
+    attr_accessor :unnamed, :parser, :dumper
+
+    def initialize(parser, dumper = nil, unnamed: false)
+      if TSV::Parser === parser
+        @parser = parser
+      elsif TSV === parser
+        @parser = parser
+      else
+        @parser = TSV::Parser.new parser
+      end
+      @unnamed = unnamed
+      if dumper.nil?
+        @dumper = TSV::Dumper.new(@parser)
+        @dumper.sep = "\t"
+      else
+        @dumper = dumper
+      end
+    end
+
+    def key_field=(key_field)
+      @dumper.key_field = key_field
+    end
+
+    def fields=(fields)
+      @dumper.fields = fields
+    end
+
+    def type=(type)
+      @dumper.type = type
+    end
+
+    def type
+      @dumper.type
+    end
+
+    def sep=(sep)
+      @dumper.sep = sep
+    end
+
+    def include?(*args)
+      false
+    end
+
+    def key_field
+      @dumper.key_field
+    end
+
+    def fields
+      @dumper.fields
+    end
+
+    def all_fields
+      return nil if fields.nil?
+      [key_field] + fields
+    end
+
+    def options
+      @dumper.options
+    end
+
+    def identify_field(name)
+      TSV.identify_field key_field, fields, name
+    end
+
+    def traverse(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+      end
+    end
+
+    def each(*args, **kwargs, &block)
+      kwargs[:into] = @dumper
+      kwargs[:bar] = "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @target}" if TrueClass === kwargs[:bar]
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      Open.traverse(@parser, *args, **kwargs) do |k,v|
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        block.call k, v
+        [k, v]
+      end
+    end
+
+    def with_unnamed
+      begin
+        old_unnamed = @unnamed
+        @unnamed = true
+        yield
+      ensure
+        @unnamed = old_unnamed
+      end
+    end
+
+    def []=(key, value)
+      @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
+      @dumper.add key, value
+    end
+
+    def stream
+      @dumper.stream
+    end
+
+    def tsv(*args)
+      TSV === @dumper ? @dumper : TSV.open(stream, *args)
+    end
+  end
+
+  def to_list
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :list
+    transformer.traverse do |k,v|
+      case self.type
+      when :single
+        [k, [v]]
+      when :double
+        [k, v.collect{|v| v.first }]
+      when :flat
+        [k, v.slice(0,1)]
+      end
+    end
+    res
+  end
+
+  def to_single
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    transformer.type = :single
+    transformer.traverse do |k,v|
+      v = v.first while Array === v
+      [k, v]
+    end
+    res
+  end
+end
+

data/lib/scout/tsv/traverse.rb

@@ -1,22 +1,14 @@
 require_relative 'parser'
 module TSV
-  def self.identify_field(key_field, fields, name)
-    return :key if name == :key || key_field.start_with?(name.to_s)
-    name.collect!{|n| key_field == n ? :key : n } if Array === name
-    NamedArray.identify_name(fields, name)
-  end
-
-  def identify_field(name)
-    TSV.identify_field(@key_field, @fields, name)
-  end
-
-  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, &block)
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, bar: false, cast: nil, select: nil, &block)
     key_field = key_field_pos if key_field.nil?
-    fields = fields_pos if fields.nil?
+    fields = fields_pos.dup if fields.nil?
     type = @type if type.nil?
     key_pos = self.identify_field(key_field)
+    fields = self.all_fields if fields == :all
     fields = [fields] unless fields.nil? || Array === fields
-    positions = fields.nil? ? nil : self.identify_field(fields)
+    positions = fields.nil? || fields == :all ? nil : self.identify_field(fields)
+
 
     if key_pos == :key
       key_name = @key_field
@@ -33,7 +25,7 @@ module TSV
       field_names = @fields
     elsif positions.nil? && key_pos != :key
       field_names = @fields.dup
-      field_names.delete_at key_pos
+      field_names.delete_at key_pos unless fields == :all
     elsif positions.include?(:key)
       field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
     else
@@ -43,98 +35,110 @@ module TSV
     key_index = positions.index :key if positions
     positions.delete :key if positions
 
-    each do |key,values|
-      values = [values] if @type == :single
-      if positions.nil?
-        if key_pos != :key
-          values = values.dup
-          key = values.delete_at(key_pos)
-        end
-      else 
-        orig_key = key
-        key = values[key_pos] if key_pos != :key 
+    log_message = "Traverse #{Log.fingerprint self}"
+    Log.debug log_message
+    bar = log_message if TrueClass === bar
 
-        values = values.values_at(*positions)
-        if key_index
-          if @type == :double
-            values.insert key_index, [orig_key]
-          else
-            values.insert key_index, orig_key
+    Log::ProgressBar.with_obj_bar(self, bar) do |bar|
+      with_unnamed unnamed do
+        each do |key,values|
+          bar.tick if bar
+          values = [values] if @type == :single
+          if positions.nil?
+            if key_pos != :key
+              values = values.dup
+              key = values.delete_at(key_pos)
+            end
+          else 
+            orig_key = key
+            key = values[key_pos] if key_pos != :key 
+
+            values = values.values_at(*positions)
+            if key_index
+              if @type == :double
+                values.insert key_index, [orig_key]
+              else
+                values.insert key_index, orig_key
+              end
+            end
           end
-        end
-      end
 
-      if Array === key 
-        if @type == :double && one2one
-          if one2one == :fill
-            key.each_with_index do |key_i,i|
-              if type == :double
-                v_i = values.collect{|v| [v[i] || v.first] }
+          values = TSV.cast_value(values, cast) if cast
+
+          if Array === key 
+            if @type == :double && one2one
+              if one2one == :strict
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i]] }
+                  else
+                    v_i = values.collect{|v| v[i] }
+                  end
+                  yield key_i, v_i
+                end
               else
-                v_i = values.collect{|v| v[i] || v.first }
+                key.each_with_index do |key_i,i|
+                  if type == :double
+                    v_i = values.collect{|v| [v[i] || v.first] }
+                  else
+                    v_i = values.collect{|v| v[i] || v.first }
+                  end
+                  yield key_i, v_i, @fields
+                end
+              end
+            else
+              key.each_with_index do |key_i, i|
+                if type == :double
+                  yield key_i, values
+                elsif type == :list
+                  yield key_i, values.collect{|v| v[i] }
+                elsif type == :flat
+                  yield key_i, values.flatten
+                elsif type == :single
+                  yield key_i, values.first
+                end
               end
-              yield key_i, v_i
             end
           else
-            key.each_with_index do |key_i,i|
-              if type == :double
-                v_i = values.collect{|v| [v[i]] }
+            if type == @type
+              if type == :single
+                yield key, values.first
               else
-                v_i = values.collect{|v| v[i] }
+                yield key, values
+              end
+            else
+              case [type, @type]
+              when [:double, :list]
+                yield key, values.collect{|v| [v] }
+              when [:double, :flat]
+                yield key, [values]
+              when [:double, :single]
+                yield key, [values]
+              when [:list, :double]
+                yield key, values.collect{|v| v.first }
+              when [:list, :flat]
+                yield key, [values.first]
+              when [:list, :single]
+                yield key, values
+              when [:flat, :double]
+                yield key, values.flatten
+              when [:flat, :list]
+                yield key, values.flatten
+              when [:flat, :single]
+                yield key, values
+              when [:single, :double]
+                yield key, values.flatten.first
+              when [:single, :list]
+                yield key, values.first
+              when [:single, :flat]
+                yield key, values.first
               end
-              yield key_i, v_i
-            end
-          end
-        else
-          key.each_with_index do |key_i, i|
-            if type == :double
-              yield key_i, values
-            elsif type == :list
-              yield key_i, values.collect{|v| v[i] }
-            elsif type == :flat
-              yield key_i, values.flatten
-            elsif type == :single
-              yield key_i, values.first
             end
           end
         end
-      else
-        if type == @type
-          if type == :single
-            yield key, values.first
-          else
-            yield key, values
-          end
-        else
-          case [type, @type]
-          when [:double, :list]
-            yield key, values.collect{|v| [v] }
-          when [:double, :flat]
-            yield key, [values]
-          when [:double, :single]
-            yield key, [values]
-          when [:list, :double]
-            yield key, values.collect{|v| v.first }
-          when [:list, :flat]
-            yield key, [values.first]
-          when [:list, :single]
-            yield key, values
-          when [:flat, :double]
-            yield key, values.flatten
-          when [:flat, :list]
-            yield key, values.flatten
-          when [:flat, :single]
-            yield key, values
-          when [:single, :double]
-            yield key, values.flatten.first
-          when [:single, :list]
-            yield key, values.first
-          when [:single, :flat]
-            yield key, values.first
-          end
-        end
       end
     end
+    
 
     [key_name, field_names]
   end

data/lib/scout/tsv/util/filter.rb

@@ -299,5 +299,14 @@ module TSV
       FileUtils.rm f
     end
   end
+
+  def with_filters(filters, &block)
+    filter
+    begin
+      filters.each{|field,value| add_filter field, value }
+    ensure
+      reset_filters
+    end
+  end
 end
 

data/lib/scout/tsv/util/reorder.rb

@@ -0,0 +1,81 @@
+require 'matrix'
+
+module TSV
+  def reorder(key_field = nil, fields = nil, merge: true, one2one: true, **kwargs) 
+    res = self.annotate({})
+    res.type = kwargs[:type] if kwargs.include?(:type)
+    kwargs[:one2one] = one2one
+    key_field_name, field_names = traverse key_field, fields, **kwargs do |k,v|
+      if @type == :double && merge && res.include?(k)
+        current = res[k]
+        if merge == :concat
+          v.each_with_index do |new,i|
+            next if new.empty?
+            current[i].concat(new)
+          end
+        else
+          merged = []
+          v.each_with_index do |new,i|
+            next if new.empty?
+            merged[i] = current[i] + new
+          end
+          res[k] = merged
+        end
+      else
+        res[k] = v
+      end
+    end
+    res.key_field = key_field_name
+    res.fields = field_names
+    res
+  end
+
+  def slice(fields, **kwargs)
+    reorder :key, fields, **kwargs
+  end
+
+  def column(field, **kwargs)
+    new_type = case type
+               when :double, :flat
+                 :flat
+               else
+                 :single
+               end
+
+    kwargs[:type] = new_type
+    slice(field, **kwargs)
+  end
+
+  def transpose_list(key_field="Unkown ID")
+    new_fields = keys.dup
+    new = self.annotate({})
+    TSV.setup(new, :key_field => key_field, :fields => new_fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    m = Matrix.rows values 
+    new_rows = m.transpose.to_a
+
+    fields.zip(new_rows) do |key,row|
+      new[key] = row
+    end
+
+    new
+  end
+
+  def transpose_double(key_field = "Unkown ID")
+    sep = "-!SEP--#{rand 10000}!-"
+    tmp = self.to_list{|v| v * sep}
+    new = tmp.transpose_list(key_field)
+    new.to_double{|v| v.split(sep)}
+  end
+
+  def transpose(key_field = "Unkown ID")
+    case type
+    when :single, :flat
+      self.to_list.transpose_list key_field
+    when :list
+      transpose_list key_field
+    when :double
+      transpose_double key_field
+    end
+  end
+end