scout-gear 7.3.0 → 8.1.0

data/lib/scout/tsv/parser.rb

@@ -12,17 +12,24 @@ module TSV
     end
   end
 
-  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil)
+  def self.parse_line(line, type: :list, key: 0, positions: nil, sep: "\t", sep2: "|", cast: nil, select: nil, field_names: nil)
     items = line.split(sep, -1)
 
+    return nil if select && ! TSV.select(items[0], items[1..-1], select, fields: field_names, type: type, sep: sep2)
+
     if positions.nil? && key == 0
       key = items.shift
-    elsif positions.nil? 
-      key = items.delete_at(key)
+    elsif positions.nil?
+      if type == :flat
+        key = items[1..-1].collect{|e| e.split(sep2, -1) }.flatten
+        items = items.slice(0,1)
+      else
+        key = items.delete_at(key)
+      end
       key = key.split(sep2) if type == :double
     else 
       key, items = items[key], items.values_at(*positions)
-      key = key.split(sep2) if type == :double
+      key = key.split(sep2) if type == :double || type == :flat
     end
 
     items = case type
@@ -31,9 +38,9 @@ module TSV
             when :single
               items.first
             when :flat
-              [items]
+              items.collect{|i| i.split(sep2, -1) }.flatten
             when :double
-              items.collect{|i| i.split(sep2, -1) }
+              items.collect{|i| i.nil? ? [] : i.split(sep2, -1) }
             end
 
 
@@ -44,21 +51,34 @@ module TSV
     [key, items]
   end
 
-  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, **kargs, &block)
+  def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, **kargs, &block)
     begin
-      bar = Log::ProgressBar.new_bar(bar) if bar
+      bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
+      bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
+      bar.init if bar
 
       source_type = type if source_type.nil?
 
       data = {} if data.nil?
-      merge = false if type != :double
+      merge = false if type != :double && type != :flat
       line = first_line || stream.gets
       while line
         begin
-          line.strip!
-          line = Misc.fixutf8(line) if fix
+          line.chomp!
+          if Proc === fix
+            line = fix.call line
+          elsif fix
+            line = Misc.fixutf8(line)
+          end
           bar.tick if bar
-          key, items = parse_line(line, type: source_type, **kargs)
+          if type == :array || type == :line
+            block.call line
+            next
+          end
+
+          key, items = parse_line(line, type: source_type, field_names: field_names, **kargs)
+
+          next if key.nil?
 
           if Array === key
             keys = key
@@ -116,44 +136,68 @@ module TSV
               end
 
             if block_given?
-              res = block.call(key, these_items)
+              res = block.call(key, these_items, field_names)
               data[key] = res unless res.nil? || FalseClass === data
               next
             end
 
             if ! merge || ! data.include?(key)
               data[key] = these_items
-            else
+            elsif type == :double
               current = data[key]
               if merge == :concat
                 these_items.each_with_index do |new,i|
-                  next if new.empty?
+                  new = [nil] if new.empty?
                   current[i].concat(new)
                 end
               else
                 merged = []
                 these_items.each_with_index do |new,i|
-                  next if new.empty?
+                  new = [nil] if new.empty?
                   merged[i] = current[i] + new
                 end
                 data[key] = merged
               end
+            elsif type == :flat
+              current = data[key]
+              if merge == :concat
+                current[i].concat these_items
+              else
+                data[key] = current + these_items
+              end
             end
           end
+        rescue Exception
+          stream.abort($!) if stream.respond_to?(:abort)
+          raise $!
         ensure
-          line = stream.gets
+          if stream.closed?
+            line = nil
+          else
+            line = stream.gets 
+          end
         end
       end
       data
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
+      if stream.stream_exception
+        bar.remove(stream.stream_exception)
+      else
+        bar.remove
+      end if bar
+      stream.join if stream.respond_to?(:join)
     end
   end
 
   def self.parse_header(stream, fix: true, header_hash: '#', sep: "\t")
+    if (Path === stream) || ((String === stream) && Path.is_filename?(stream))
+      Open.open(stream) do |f|
+        return parse_header(f, fix: fix, header_hash: header_hash, sep: sep)
+      end
+    end
     raise "Closed stream" if IO === stream && stream.closed?
 
-    options = {}
+    opts = {}
     preamble = []
 
     # Get line
@@ -165,13 +209,19 @@ module TSV
 
     # Process options line
     if line and (String === header_hash && m = line.match(/^#{header_hash}: (.*)/))
-      options = IndiferentHash.string2hash m.captures.first.chomp
+      opts = IndiferentHash.string2hash m.captures.first.chomp
       line = stream.gets
-      line = Misc.fixutf8 line.chomp if line && fix
+      if line && fix
+        if Proc === fix
+          line = fix.call line
+        else
+          line = Misc.fixutf8 line.chomp if line && fix
+        end
+      end
     end
 
     # Determine separator
-    sep = options[:sep] if options[:sep]
+    sep = opts[:sep] if opts[:sep]
 
     # Process fields line
     preamble << line if line
@@ -192,7 +242,10 @@ module TSV
 
     first_line = line
 
-    [options, key_field, fields, first_line, preamble]
+    opts[:type] = opts[:type].to_sym if opts[:type]
+    opts[:cast] = opts[:cast].to_sym if opts[:cast]
+
+    NamedArray.setup([opts, key_field, fields, first_line, preamble], %w(options key_field fields first_line preamble))
   end
 
   KEY_PARAMETERS = begin
@@ -204,43 +257,81 @@ module TSV
                    end
 
   class Parser
-    attr_accessor :stream, :options, :key_field, :fields, :first_line, :preamble
-    def initialize(file, fix: true, header_hash: "#", sep: "\t")
+    attr_accessor :stream, :options, :key_field, :fields, :type, :first_line, :preamble
+    def initialize(file, fix: true, header_hash: "#", sep: "\t", type: :double)
       if IO === file
         @stream = file
       else
         @stream = Open.open(file)
       end
+      @fix = fix
       @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
       @options[:sep] = sep if @options[:sep].nil?
+      @options.merge!(:key_field => @key_field, :fields => @fields)
+      @type = @options[:type] || type
     end
 
     def all_fields
+      return nil if @fields.nil?
       [@key_field] + @fields
     end
 
+    def key_field=(key_field)
+      @options[:key_field] = @key_field = key_field
+    end
+    
+    def fields=(fields)
+      @options[:fields] = @fields = fields
+    end
+
+    def identify_field(name)
+      TSV.identify_field(@key_field, @fields, name)
+    end
+
     def traverse(key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
+      kwargs[:type] ||=  self.options[:type] ||= @type
+      kwargs[:type] = kwargs[:type].to_sym if kwargs[:type]
+
       if fields
-        all_field_names ||= [@key_field] + @fields
-        positions = NamedArray.identify_name(all_field_names, fields)
-        kwargs[:positions] = positions
-        field_names = all_field_names.values_at *positions
+        if @fields
+          all_field_names ||= [@key_field] + @fields
+          fields = all_field_names if fields == :all
+          positions = NamedArray.identify_name(all_field_names, fields)
+          kwargs[:positions] = positions
+          field_names = all_field_names.values_at *positions
+        elsif fields.reject{|f| Numeric === f}.empty?
+          positions = fields
+          kwargs[:positions] = positions
+        else
+          raise "Non-numeric fields specified, but no field names available"
+        end
       else
         field_names = @fields
       end
 
+      kwargs[:positions] = nil if @type == :flat
+
       if key_field
-        all_field_names ||= [@key_field] + @fields
-        key = NamedArray.identify_name(all_field_names, key_field)
-        kwargs[:key] = key == :key ? 0 : key
-        key_field_name = key === :key ? @key_field : all_field_names[key]
-        if fields.nil?
-          field_names = all_field_names - [@key_field]
+        if @fields
+          all_field_names ||= [@key_field] + @fields
+          key = NamedArray.identify_name(all_field_names, key_field)
+          kwargs[:key] = key == :key ? 0 : key
+          key_field_name = key === :key ? @key_field : all_field_names[key]
+          if fields.nil?
+            field_names = all_field_names - [key_field_name]
+          end
+        else
+          kwargs[:key] = key_field == :key ? 0 : key_field
+          key = key_field
         end
       else
         key_field_name = @key_field
       end
 
+      if field_names && (kwargs[:type] == :single || kwargs[:type] == :flat)
+        field_names = field_names.slice(0,1)
+      end
+
       @options.each do |option,value|
         option = option.to_sym
         next unless KEY_PARAMETERS.include? option
@@ -250,21 +341,33 @@ module TSV
       kwargs[:source_type] = @options[:type]
       kwargs[:data] = false if kwargs[:data].nil?
 
-      data = TSV.parse_stream(@stream, first_line: @first_line, **kwargs, &block)
+      data = TSV.parse_stream(@stream, first_line: @first_line, fix: @fix, field_names: @fields, **kwargs, &block)
 
       if data
         TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => @type)
       else
-        self
+        [self.key_field, self.fields]
       end
     end
 
+    def fingerprint
+      "Parser:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+    end
+
+    def digest_str
+      fingerprint
+    end
+
+    def inspect
+      fingerprint
+    end
   end
 
   def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
     parser = TSV::Parser.new stream, fix: fix, header_hash: header_hash, sep: sep
 
-    cast = parser.options[:cast] || kwargs[:cast]
+    cast = kwargs[:cast]
+    cast = parser.options[:cast] if cast.nil?
     type = kwargs[:type] ||=  parser.options[:type] ||= :double
     if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
       TSV.setup(data, type: type)
@@ -297,48 +400,7 @@ module TSV
     data.filename = filename
     data.namespace = namespace
     data.unnamed = unnamed
+    data.save_extension_attr_hash if data.respond_to?(:save_extension_attr_hash)
     data
   end
-
-  #def self.parse_alt(stream, key_field: nil, fields: nil, filename: nil, namespace: nil,  **kwargs, &block)
-  #  options, key_field_name, field_names, first_line, preamble = parse_header(stream)
-
-  #  if fields
-  #    all_field_names ||= [key_field_name] + field_names
-  #    positions = NamedArray.identify_name(all_field_names, fields)
-  #    kwargs[:positions] = positions
-  #    field_names = all_field_names.values_at *positions
-  #  end
-
-  #  if key_field
-  #    all_field_names ||= [key_field_name] + field_names
-  #    key = NamedArray.identify_name(all_field_names, key_field)
-  #    kwargs[:key] = key
-  #    key_field_name = all_field_names[key]
-  #    if fields.nil?
-  #      field_names = all_field_names - [key_field_name]
-  #    end
-  #  end
-
-  #  options.each do |option,value|
-  #    option = option.to_sym
-  #    next unless KEY_PARAMETERS.include? option
-  #    kwargs[option] = value unless kwargs.include?(option)
-  #  end
-
-  #  kwargs[:source_type] = options[:type]
-
-  #  type = kwargs[:type] ||= :double
-  #  if (data = kwargs[:data]) && data.respond_to?(:persistence_class)
-  #    TSV.setup(data, type: type, key_field: key_field_name, fields: field_names)
-  #    data.extend TSVAdapter
-  #  end
-
-  #  data = parse_stream(stream, first_line: first_line, **kwargs, &block)
-
-  #  TSV.setup(data, :key_field => key_field_name, :fields => field_names, :type => type, filename: filename, namespace: namespace)
-
-  #  data
-  #end
-
 end

data/lib/scout/tsv/path.rb

@@ -1,7 +1,6 @@
 module Path
   def tsv(*args, **kwargs, &block)
-    found = self.find
-    found = self.set_extension('tsv').find unless found.exists?
+    found = produce_and_find('tsv')
     TSV.open(found, *args, **kwargs, &block)
   end
 

data/lib/scout/tsv/persist/adapter.rb

@@ -1,8 +1,9 @@
 require_relative '../../open/lock'
+require_relative '../../meta_extension'
 require_relative 'serialize'
 
 module TSVAdapter
-  attr_accessor :persistence_path, :persistence_class, :serializer, :closed, :writable
+  attr_accessor :persistence_path, :persistence_class, :closed, :writable, :serializer
 
   class << self
     attr_accessor :lock_dir
@@ -19,23 +20,15 @@ module TSVAdapter
   end
 
   def load_extension_attr_hash
-    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self[EXTENSION_ATTR_HASH_KEY]))
+    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(EXTENSION_ATTR_HASH_KEY)))
   end
 
   def save_extension_attr_hash
-    self[EXTENSION_ATTR_HASH_KEY]= EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash)
+    self.orig_set(EXTENSION_ATTR_HASH_KEY, EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash))
   end
 
   def self.extended(base)
-    if ! TSVAdapter === base
-      if (! TSVAdapter === base) && base.include?(EXTENSION_ATTR_HASH_KEY)
-        TSV.setup(base, base.load_extension_attr_hash)
-      elsif TSV === base
-        base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
-      end
-    end
-
-    base.serializer = SERIALIZER_ALIAS[base.type]
+    return if base.respond_to?(:orig_set)
 
     class << base
       alias orig_set []=
@@ -54,45 +47,22 @@ module TSVAdapter
       end
 
       def load_value(str)
+        return nil if str.nil?
         serializer.load(str)
       end
 
       def save_value(value)
         serializer.dump(value)
       end
-
-
     end
 
-    #case base.type
-    #when :single
-    #  class << base
-    #    def load_value(value)
-    #      value
-    #    end
-    #    def save_value(value)
-    #      value
-    #    end
-    #  end
-    #when :list, :flat
-    #  class << base
-    #    def load_value(value)
-    #      value.nil? ? nil : value.split("\t")
-    #    end
-    #    def save_value(value)
-    #      value * "\t"
-    #    end
-    #  end
-    #when :double
-    #  class << base
-    #    def load_value(value)
-    #      value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
-    #    end
-    #    def save_value(value)
-    #      value.collect{|v| v * "|" } * "\t"
-    #    end
-    #  end
-    #end
+    if base.include?(EXTENSION_ATTR_HASH_KEY)
+      TSV.setup(base, base.load_extension_attr_hash)
+    else
+      base.instance_variable_get(:@extension_attrs).push :serializer
+      base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
+      base.save_extension_attr_hash
+    end
   end
 
   def keys(*args)
@@ -101,9 +71,9 @@ module TSVAdapter
     end
 
     if k[0] == EXTENSION_ATTR_HASH_KEY
-      k.slice(1,k.length-1)
+      k.slice(1,k.length)
     elsif k[-1] == EXTENSION_ATTR_HASH_KEY
-      k.slice(0,k.length-2)
+      k.slice(0,k.length-1)
     else
       k - [EXTENSION_ATTR_HASH_KEY]
     end

data/lib/scout/tsv/persist/fix_width_table.rb

@@ -2,6 +2,7 @@ class FixWidthTable
 
   attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
+    filename = filename.find if Path === filename
     @filename = filename
 
     if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
@@ -312,6 +313,8 @@ class FixWidthTable
       acc.concat(new)
     end
   end
+
+  alias length size
 end
 
 Persist.save_drivers[:fwt] = proc do |file, content|

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -16,10 +16,13 @@ module ScoutCabinet
     dir = File.dirname(File.expand_path(path))
     Open.mkdir(dir) unless File.exist?(dir)
 
+    tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
     tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
     tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    database = Persist::CONNECTIONS[path] ||= tokyocabinet_class.new
+    # Hack - Ignore warning: undefining the allocator of T_DATA class
+    # TokyoCabinet::HDB_data
+    database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
     if big and not Open.exists?(path)
       database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
@@ -39,6 +42,8 @@ module ScoutCabinet
 
     database.open(path, tokyocabinet_class::OREADER)
 
+    database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
+
     Persist::CONNECTIONS[path] = database
 
     database

data/lib/scout/tsv/persist.rb

@@ -6,6 +6,10 @@ Persist.save_drivers[:tsv] = proc do |file,content|
              content
            elsif content.respond_to?(:stream)
              content.stream
+           elsif content.respond_to?(:dumper_stream)
+             content.dumper_stream
+           else
+             content
            end
   Open.sensible_write(file, stream)
 end