scout-gear 7.2.0 → 7.3.0

data/lib/scout/tsv/index.rb

@@ -1,6 +1,7 @@
 require_relative 'parser'
+require_relative 'persist/fix_width_table'
 module TSV
-  def self.index(tsv_file, target: 0, order: true, **kwargs)
+  def self.index(tsv_file, target: 0, fields: nil, order: true, **kwargs)
     persist, type = IndiferentHash.process_options kwargs,
       :persist, :persist_type,
       :persist => false, :persist_type => "HDB"
@@ -15,17 +16,15 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      dummy_data = nil
+      dummy_data = TSV.setup({}, :key_field => "Key", :fields => ["Target"])
       if order
         tmp_index = {}
-        dummy_data = Open.open(tsv_file) do |file|
-          TSV.parse file, key_field: target, type: :double, **kwargs do |k,values|
-            values.each_with_index do |list,i|
-              list.each do |e|
-                tmp_index[e] ||= []
-                tmp_index[e][i] ||= []
-                tmp_index[e][i] << k
-              end
+        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :double, into: dummy_data, unnamed: true, **kwargs do |k,values|
+          values.each_with_index do |list,i|
+            list.each do |e|
+              tmp_index[e] ||= []
+              tmp_index[e][i] ||= []
+              tmp_index[e][i] << k
             end
           end
         end
@@ -33,17 +32,130 @@ module TSV
           index[e] = list.flatten.compact.uniq.first
         end
       else
-        dummy_data = Open.open(tsv_file) do |file|
-          TSV.parse file, key_field: target, type: :flat, **kwargs do |k,values|
-            values.each do |e|
-              index[e] = k unless index.include?(e)
-            end
+        key_field, field_names = TSV.traverse tsv_file, key_field: target, fields: fields, type: :flat, into: dummy_data, unnamed: true, **kwargs do |k,values|
+          values.each do |e|
+            index[e] = k unless index.include?(e)
           end
         end
       end
+
       index.key_field = dummy_data.fields * ", "
       index.fields = [dummy_data.key_field]
       index
     end
   end
+
+  def index(*args, **kwargs, &block)
+    TSV.index(self, *args, **kwargs, &block)
+  end
+
+  def self.range_index(tsv_file, start_field = nil, end_field = nil, key_field: :key, **kwargs)
+    persist, type = IndiferentHash.process_options kwargs,
+      :persist, :persist_type,
+      :persist => false, :persist_type => :fwt
+    kwargs.delete :type
+
+    Persist.persist(tsv_file, type, kwargs.merge(:persist => persist, :persist_prefix => "Index")) do |filename|
+
+      max_key_size = 0
+      index_data = []
+      TSV.traverse tsv_file, key_field: key_field, fields: [start_field, end_field] do |key, values|
+        key_size = key.length
+        max_key_size = key_size if key_size > max_key_size
+
+        start_pos, end_pos = values
+        if Array === start_pos
+          start_pos.zip(end_pos).each do |s,e|
+            index_data << [key, [s.to_i, e.to_i]]
+          end
+        else
+          index_data << [key, [start_pos.to_i, end_pos.to_i]]
+        end
+      end
+
+      filename = :memory if filename.nil?
+      index = FixWidthTable.get(filename, max_key_size, true)
+      index.add_range index_data
+      index.read
+      index
+    end
+  end
+
+  def range_index(*args, **kwargs, &block)
+    TSV.range_index(self, *args, **kwargs, &block)
+  end
+
+
+  #def range_index(start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  options = Misc.add_defaults options,
+  #    :persist => false, :persist_file => nil, :persist_update => false 
+
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "RangeIndex[#{start_field}-#{end_field}]"
+
+  #  Persist.persist(filename || self.object_id.to_s, :fwt, persist_options) do 
+  #    max_key_size = 0
+  #    index_data = []
+  #    with_unnamed do
+  #      with_monitor :desc => "Creating Index Data", :step => 10000 do
+  #        through :key, [start_field, end_field] do |key, values|
+  #          key_size = key.length
+  #          max_key_size = key_size if key_size > max_key_size
+
+  #          start_pos, end_pos = values
+  #          if Array === start_pos
+  #            start_pos.zip(end_pos).each do |s,e|
+  #              index_data << [key, [s.to_i, e.to_i]]
+  #            end
+  #          else
+  #            index_data << [key, [start_pos.to_i, end_pos.to_i]]
+  #          end
+  #        end
+  #      end
+  #    end
+
+  #    index = FixWidthTable.get(:memory, max_key_size, true)
+  #    index.add_range index_data
+  #    index.read
+  #    index
+  #  end
+  #end
+
+  #def self.range_index(file, start_field = nil, end_field = nil, options = {})
+  #  start_field ||= "Start"
+  #  end_field ||= "End"
+
+  #  data_options = Misc.pull_keys options, :data
+  #  filename = case
+  #             when (String === file or Path === file)
+  #               file
+  #             when file.respond_to?(:filename)
+  #               file.filename
+  #             else
+  #               file.object_id.to_s
+  #             end
+  #  persist_options = Misc.pull_keys options, :persist
+  #  persist_options[:prefix] ||= "StaticRangeIndex[#{start_field}-#{end_field}]"
+
+  #  filters = Misc.process_options options, :filters
+
+  #  if filters
+  #    filename += ":Filtered[#{filters.collect{|f| f * "="} * ", "}]"
+  #  end
+
+  #  Persist.persist(filename, :fwt, persist_options) do
+  #    tsv = TSV.open(file, data_options)
+  #    if filters
+  #      tsv.filter
+  #      filters.each do |match, value|
+  #        tsv.add_filter match, value
+  #      end
+  #    end
+ 
+  #    tsv.range_index(start_field, end_field, options)
+  #  end
+  #end
 end

data/lib/scout/tsv/open.rb

@@ -0,0 +1,128 @@
+require_relative '../open'
+module Open
+  def self.traverse_add(into, res)
+    case into
+    when TSV::Dumper
+      into.add *res
+    when TSV, Hash
+      key, value = res
+      into[key] = value
+    end
+  end
+
+  #def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
+  #  case obj
+  #  when TSV
+  #    obj.traverse options[:key_field], options[:fields], **options do |k,v|
+  #      res = yield k, v
+  #    end
+  #  when String
+  #    f = Open.open(obj)
+  #    self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
+  #  when Step
+  #    self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
+  #  when IO
+  #    if into && (IO === into || into.respond_to?(:stream) )
+  #      into_thread = Thread.new do 
+  #        Thread.current.report_on_exception = false
+  #        Thread.current["name"] = "Traverse into"
+  #        TSV.parse obj, **options do |k,v|
+  #          begin
+  #            res = block.call k, v
+  #            traverse_add into, res
+  #          rescue
+  #            into.abort $!
+  #          end
+  #          nil
+  #        end
+  #        into.close if into.respond_to?(:close)
+  #      end
+  #      Thread.pass until into_thread
+  #      into
+  #    else
+  #      TSV.parse obj, **options do |k,v|
+  #        block.call k, v
+  #        nil
+  #      end
+  #    end
+  #  end
+  #end
+
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, callback: nil, unnamed: true, **options, &block)
+
+    if into || bar
+      orig_callback = callback if callback
+      bar = Log::ProgressBar.get_obj_bar(bar, obj)
+      callback = proc do |res|
+        bar.tick if bar
+        traverse_add into, res if into
+        orig_callback.call res if orig_callback
+      end
+
+      if into.respond_to?(:close)
+        into_thread = Thread.new do 
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Traverse into"
+          error = false
+          begin
+            self.traverse(obj, callback: callback, **options, &block)
+            into.close if into.respond_to?(:close)
+            bar.remove if bar
+          rescue Exception
+            into.abort($!) if into.respond_to?(:abort)
+            bar.remove($!) if bar
+          end
+        end
+        Thread.pass until into_thread
+        return into
+      end
+    end
+
+    begin
+      case obj
+      when TSV
+        obj.traverse options[:key_field], options[:fields], unnamed: unnamed, **options do |k,v|
+          res = block.call(k, v)
+          callback.call res if callback
+          nil
+        end
+      when Array
+        obj.each do |line|
+          res = block.call(line)
+          callback.call res if callback
+          nil
+        end
+      when String
+        f = Open.open(obj)
+        self.traverse(f, cpus: cpus, callback: callback, **options, &block)
+      when Step
+        raise obj.exception if obj.error?
+        self.traverse(obj.stream, cpus: cpus, callback: callback, **options, &block)
+      when IO
+        TSV.parse obj, **options do |k,v|
+          res = block.call k, v
+          callback.call res if callback
+          nil
+        end
+      else
+        TSV.parse obj, **options do |k,v|
+          res = block.call k, v
+          callback.call res if callback
+          nil
+        end
+      end
+      bar.remove if bar
+    rescue
+      bar.abort($!) if bar
+      raise $!
+    end
+
+    into
+  end
+end
+
+module TSV
+  def self.traverse(*args, **kwargs, &block)
+    Open.traverse(*args, **kwargs, &block)
+  end
+end

data/lib/scout/tsv/parser.rb

@@ -4,7 +4,11 @@ module TSV
     if Array === value
       value.collect{|e| cast_value(e, cast) }
     else
-      value.send(cast)
+      if Proc === cast
+        cast.call value
+      else
+        value.send(cast)
+      end
     end
   end
 
@@ -75,40 +79,41 @@ module TSV
               these_items = items
             end
 
-            these_items = case [source_type, type]
-                          when [:single, :single]
-                            these_items
-                          when [:list, :single]
-                            these_items.first
-                          when [:flat, :single]
-                            these_items.first
-                          when [:double, :single]
-                            these_items.first.first
-                          when [:single, :list]
-                            [these_items]
-                          when [:list, :list]
-                            these_items
-                          when [:flat, :list]
-                            these_items
-                          when [:double, :list]
-                            these_items.collect{|l| l.first }
-                          when [:single, :flat]
-                            [these_items]
-                          when [:list, :flat]
-                            these_items
-                          when [:flat, :flat]
-                            these_items
-                          when [:double, :flat]
-                            these_items.flatten
-                          when [:single, :double]
-                            [[these_items]]
-                          when [:list, :double]
-                            these_items.collect{|l| [l] }
-                          when [:flat, :double]
-                            [these_items]
-                          when [:double, :double]
-                            these_items
-                          end
+            these_items = 
+              case [source_type, type]
+              when [:single, :single]
+                these_items
+              when [:list, :single]
+                these_items.first
+              when [:flat, :single]
+                these_items.first
+              when [:double, :single]
+                these_items.first.first
+              when [:single, :list]
+                [these_items]
+              when [:list, :list]
+                these_items
+              when [:flat, :list]
+                these_items
+              when [:double, :list]
+                these_items.collect{|l| l.first }
+              when [:single, :flat]
+                [these_items]
+              when [:list, :flat]
+                these_items
+              when [:flat, :flat]
+                these_items
+              when [:double, :flat]
+                these_items.flatten
+              when [:single, :double]
+                [[these_items]]
+              when [:list, :double]
+                these_items.collect{|l| [l] }
+              when [:flat, :double]
+                [these_items]
+              when [:double, :double]
+                these_items
+              end
 
             if block_given?
               res = block.call(key, these_items)
@@ -227,8 +232,8 @@ module TSV
       if key_field
         all_field_names ||= [@key_field] + @fields
         key = NamedArray.identify_name(all_field_names, key_field)
-        kwargs[:key] = key
-        key_field_name = all_field_names[key]
+        kwargs[:key] = key == :key ? 0 : key
+        key_field_name = key === :key ? @key_field : all_field_names[key]
         if fields.nil?
           field_names = all_field_names - [@key_field]
         end
@@ -247,21 +252,42 @@ module TSV
 
       data = TSV.parse_stream(@stream, first_line: @first_line, **kwargs, &block)
 
-      TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type) if data
-
-      data || self
+      if data
+        TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
+      else
+        self
+      end
     end
 
   end
 
-  def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil,  **kwargs, &block)
+  def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
     parser = TSV::Parser.new stream, fix: fix, header_hash: header_hash, sep: sep
-    kwargs = parser.options.merge(kwargs)
 
-    type = kwargs[:type] ||= :double
+    cast = parser.options[:cast] || kwargs[:cast]
+    type = kwargs[:type] ||=  parser.options[:type] ||= :double
     if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
       TSV.setup(data, type: type)
       data.extend TSVAdapter
+      if serializer
+        data.serializer = serializer
+      elsif cast
+        data.serializer = 
+          case [cast, type]
+          when [:to_i, :single]
+            :integer
+          when [:to_i, :list], [:to_i, :flat]
+            :integer_array
+          when [:to_f, :single]
+            :float
+          when [:to_f, :list], [:to_f, :flat]
+            :float_array
+          else
+            type
+          end
+      else
+        data.serializer = type
+      end
     end
 
     kwargs[:data] = {} if kwargs[:data].nil?
@@ -270,6 +296,7 @@ module TSV
     data.type = type
     data.filename = filename
     data.namespace = namespace
+    data.unnamed = unnamed
     data
   end
 

data/lib/scout/tsv/path.rb

@@ -1,13 +1,13 @@
 module Path
-  def tsv(...)
+  def tsv(*args, **kwargs, &block)
     found = self.find
     found = self.set_extension('tsv').find unless found.exists?
-    TSV.open(found, ...)
+    TSV.open(found, *args, **kwargs, &block)
   end
 
-  def index(...)
+  def index(*args, **kwargs, &block)
     found = self.find
     found = self.set_extension('tsv').find unless found.exists?
-    TSV.index(found, ...)
+    TSV.index(found, *args, **kwargs, &block)
   end
 end

data/lib/scout/tsv/persist/adapter.rb

@@ -1,7 +1,8 @@
 require_relative '../../open/lock'
+require_relative 'serialize'
 
 module TSVAdapter
-  attr_accessor :persistence_path, :persistence_class, :closed, :writable
+  attr_accessor :persistence_path, :persistence_class, :serializer, :closed, :writable
 
   class << self
     attr_accessor :lock_dir
@@ -13,8 +14,12 @@ module TSVAdapter
   EXTENSION_ATTR_HASH_KEY = "__extension_attr_hash__"
   EXTENSION_ATTR_HASH_SERIALIZER = Marshal
 
+  def serializer=(serializer)
+    @serializer = Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
+  end
+
   def load_extension_attr_hash
-    EXTENSION_ATTR_HASH_SERIALIZER.load(self[EXTENSION_ATTR_HASH_KEY])
+    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self[EXTENSION_ATTR_HASH_KEY]))
   end
 
   def save_extension_attr_hash
@@ -22,12 +27,16 @@ module TSVAdapter
   end
 
   def self.extended(base)
-    if base.include?(EXTENSION_ATTR_HASH_KEY)
-      TSV.setup(base, base.load_extension_attr_hash)
-    elsif TSV === base
-      base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
+    if ! TSVAdapter === base
+      if (! TSVAdapter === base) && base.include?(EXTENSION_ATTR_HASH_KEY)
+        TSV.setup(base, base.load_extension_attr_hash)
+      elsif TSV === base
+        base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
+      end
     end
 
+    base.serializer = SERIALIZER_ALIAS[base.type]
+
     class << base
       alias orig_set []=
         alias orig_get []
@@ -43,37 +52,47 @@ module TSVAdapter
           super(key, save_value(value))
         end
       end
-    end
 
-    case base.type
-    when :single
-      class << base
-        def load_value(value)
-          value
-        end
-        def save_value(value)
-          value
-        end
-      end
-    when :list, :flat
-      class << base
-        def load_value(value)
-          value.nil? ? nil : value.split("\t")
-        end
-        def save_value(value)
-          value * "\t"
-        end
+      def load_value(str)
+        serializer.load(str)
       end
-    when :double
-      class << base
-        def load_value(value)
-          value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
-        end
-        def save_value(value)
-          value.collect{|v| v * "|" } * "\t"
-        end
+
+      def save_value(value)
+        serializer.dump(value)
       end
+
+
     end
+
+    #case base.type
+    #when :single
+    #  class << base
+    #    def load_value(value)
+    #      value
+    #    end
+    #    def save_value(value)
+    #      value
+    #    end
+    #  end
+    #when :list, :flat
+    #  class << base
+    #    def load_value(value)
+    #      value.nil? ? nil : value.split("\t")
+    #    end
+    #    def save_value(value)
+    #      value * "\t"
+    #    end
+    #  end
+    #when :double
+    #  class << base
+    #    def load_value(value)
+    #      value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
+    #    end
+    #    def save_value(value)
+    #      value.collect{|v| v * "|" } * "\t"
+    #    end
+    #  end
+    #end
   end
 
   def keys(*args)