scout-gear 7.2.0 → 7.3.0

data/lib/scout/tsv/persist/fix_width_table.rb

@@ -0,0 +1,324 @@
+class FixWidthTable
+
+  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
+  def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
+    @filename = filename
+
+    if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
+      Log.debug "FixWidthTable create: #{ filename }"
+      @value_size  = value_size
+      @range       = range
+      @record_size = @value_size + (@range ? 16 : 8)
+      @write = true
+
+      if %w(memory stringio).include?(filename.to_s.downcase)
+        @filename = :memory
+        @file     = StringIO.new
+      else
+        FileUtils.rm @filename if File.exist? @filename
+        FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
+        @file = File.open(@filename, 'w:ASCII-8BIT')
+      end
+
+      @file.write [value_size].pack("L")
+      @file.write [@range ? 1 : 0 ].pack("C")
+
+      @size = 0
+    else
+      Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
+      if in_memory
+        @file        = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
+      else
+        @file        = File.open(@filename, 'r:ASCII-8BIT')
+      end
+      @value_size  = @file.read(4).unpack("L").first
+      @range       = @file.read(1).unpack("C").first == 1
+      @record_size = @value_size + (@range ? 16 : 8)
+      @write = false
+
+      @size        = (File.size(@filename) - 5) / (@record_size)
+    end
+
+    @mask = "a#{@value_size}"
+  end
+
+  def write?
+    @write
+  end
+
+  def persistence_path
+    @filename
+  end
+
+  def persistence_path=(value)
+    @filename=value
+  end
+
+  def self.get(filename, value_size = nil, range = nil, update = false)
+    return self.new(filename, value_size, range, update) if filename == :memory
+    case
+    when (!File.exist?(filename) or update or not Persist::CONNECTIONS.include?(filename))
+      Persist::CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    end
+
+    Persist::CONNECTIONS[filename] 
+  end
+
+  def format(pos, value)
+    padding = value_size - value.length
+    if @range
+      (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
+    else
+      [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
+    end
+  end
+
+  def add(pos, value)
+    format = format(pos, value)
+    @file.write format
+
+    @size += 1
+  end
+
+  def last_pos
+    pos(size - 1)
+  end
+
+  def idx_pos(index)
+    return nil if index < 0 or index >= size
+    @file.seek(5 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_pos_end(index)
+    return nil if index < 0 or index >= size
+    @file.seek(9 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_overlap(index)
+    return nil if index < 0 or index >= size
+    @file.seek(13 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_value(index)
+    return nil if index < 0 or index >= size
+    @file.seek((@range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    padding = @file.read(4).unpack("l").first+1
+    txt = @file.read(value_size)
+    str = txt.unpack(mask).first
+    padding > 1 ? str[0..-padding] : str
+  end
+
+  def read(force = false)
+    return if @filename == :memory
+    @write = false
+    @file.close unless @file.closed?
+    @file = File.open(filename, 'r:ASCII-8BIT')
+  end
+
+  def close
+    @write = false
+    @file.close
+  end
+
+  def dump
+    read
+    @file.rewind
+    @file.read
+  end
+
+  #{{{ Adding data
+
+  def add_point(data)
+    data.sort_by{|value,pos| pos }.each do |value, pos|
+      add pos, value
+    end
+  end
+
+  def add_range_point(pos, value)
+    @latest ||= []
+    while @latest.any? and @latest[0] < pos[0]
+      @latest.shift
+    end
+    overlap = @latest.length
+    add pos + [overlap], value
+    @latest << pos[1]
+  end
+
+  def add_range(data)
+    @latest = []
+    data.sort_by{|value, pos| pos[0] }.each do |value, pos|
+      add_range_point(pos, value)
+    end
+  end
+
+  #{{{ Searching
+
+  def closest(pos)
+    upper = size - 1
+    lower = 0
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = lower + (upper - lower) / 2
+      pos_idx = idx_pos(idx)
+
+      case pos <=> pos_idx
+      when 0
+        break
+      when -1
+        upper = idx - 1
+      when 1
+        lower = idx + 1
+      end
+    end
+
+    if pos_idx > pos
+      idx = idx - 1
+    end
+
+    idx.to_i
+  end
+
+  def get_range(pos, return_idx = false)
+    case pos
+    when Range
+      r_start = pos.begin
+      r_end   = pos.end
+    when Array
+      r_start, r_end = pos
+    else
+      r_start, r_end = pos, pos
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx < 0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    overlap = idx_overlap(idx)
+
+    idx -= overlap unless overlap.nil?
+
+    values = []
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    
+    if return_idx
+      while l_start <= r_end
+        values << idx if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx) if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    end
+
+    values
+  end
+
+  def get_point(pos, return_idx = false)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx < 0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx += 1 unless idx_pos(idx) >= r_start
+
+    return [] if idx >= size
+
+    values = []
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    if return_idx 
+      while l_start <= r_end
+        values << idx
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx)
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    end
+
+    values
+  end
+
+  def [](pos)
+    return [] if size == 0
+    self.read
+    if @range
+      get_range(pos)
+    else
+      get_point(pos)
+    end
+  end
+  
+  def overlaps(pos, value = false)
+    return [] if size == 0
+    idxs = if @range
+      get_range(pos, true)
+    else
+      get_point(pos, true)
+    end
+    if value
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx), idx_value(idx)] * ":"}
+    else
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx)] * ":"}
+    end
+  end
+
+
+  def values_at(*list)
+    list.collect{|pos|
+      self[pos]
+    }
+  end
+
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
+end
+
+Persist.save_drivers[:fwt] = proc do |file, content|
+  content.file.seek 0
+  Misc.sensiblewrite(file, content.file.read)
+end
+
+Persist.load_drivers[:fwt] = proc do |file| 
+  FixWidthTable.new file
+end

data/lib/scout/tsv/persist/serialize.rb

@@ -0,0 +1,117 @@
+require 'base64'
+module TSVAdapter
+
+  class CleanSerializer
+    def self.dump(o); o end
+    def self.load(o); o end
+  end
+
+  class BinarySerializer
+    def self.dump(o); [o].pack('m'); end
+    def self.load(str); str.unpack('m').first; end
+  end
+  
+  class IntegerSerializer
+    def self.dump(i); [i].pack("l"); end
+    def self.load(str); str.unpack("l").first; end
+  end
+
+  class FloatSerializer
+    def self.dump(i); [i].pack("d"); end
+    def self.load(str); str.unpack("d").first; end
+  end
+
+  class StrictIntegerArraySerializer
+    def self.dump(a); a.pack("l*"); end
+    def self.load(str); a = str.unpack("l*"); end
+  end
+
+  class StrictFloatArraySerializer
+    def self.dump(a); a.pack("d*"); end
+    def self.load(str); a = str.unpack("d*"); end
+  end
+
+  class IntegerArraySerializer
+    NIL_INT = -999
+    def self.dump(a); a.collect{|v| v || NIL_INT}.pack("l*"); end
+    def self.load(str); a = str.unpack("l*"); a.collect{|v| v == NIL_INT ? nil : v}; end
+  end
+
+  class FloatArraySerializer
+    NIL_FLOAT = -999.999
+    def self.dump(a); a.collect{|v| v || NIL_FLOAT}.pack("d*"); end
+    def self.load(str); a = str.unpack("d*"); a.collect{|v| v == NIL_FLOAT ? nil : v}; end
+  end
+
+  class StringSerializer
+    def self.dump(str); str.to_s; end
+    def self.load(str); str.dup; end
+  end
+
+  class StringArraySerializer
+    def self.dump(array)
+      array.collect{|a| a.to_s} * "\t"
+    end
+
+    def self.load(string)
+      return nil if string.nil? or string == 'nil'
+      return [] if string.empty?
+      string.split("\t", -1)
+    end
+  end
+
+  class StringDoubleArraySerializer
+    def self.dump(array)
+      begin
+      array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
+      rescue Encoding::CompatibilityError
+        array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
+      end
+    end
+
+    def self.load(string)
+      return [] if string.nil?
+      string.split("\t", -1).collect{|l| l.split("|", -1)}
+    end
+  end
+
+  class TSVMarshalSerializer
+    def self.dump(tsv)
+      Marshal.dump(tsv.dup)
+    end
+
+    def self.load(string)
+      TSV.setup Marshal.load(string)
+    end
+  end
+
+  class TSVSerializer
+    def self.dump(tsv)
+      tsv.to_s
+    end
+
+    def self.load(string)
+      TSV.open StringIO.new(string)
+    end
+  end
+
+  SERIALIZER_ALIAS = {
+    :single => StringSerializer,
+    :list => StringArraySerializer,
+    :flat => StringArraySerializer,
+    :double => StringDoubleArraySerializer,
+    :clean => CleanSerializer,
+    :integer => IntegerSerializer, 
+    :float => FloatSerializer, 
+    :integer_array => IntegerArraySerializer,
+    :float_array => FloatArraySerializer,
+    :strict_integer_array => StrictIntegerArraySerializer,
+    :strict_float_array => StrictFloatArraySerializer,
+    :marshal => Marshal,
+    :string => StringSerializer,
+    :binary => BinarySerializer,
+    :tsv => TSVSerializer,
+    :marshal_tsv => TSVMarshalSerializer
+  }
+
+end

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -4,8 +4,8 @@ require_relative 'adapter'
 module ScoutCabinet
   attr_accessor :persistence_path, :persistence_class
 
-  CONNECTIONS = {}
   def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+    path = path.find if Path === path
     if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
       big = true
       tokyocabinet_class = tokyocabinet_class.split(":").first
@@ -19,7 +19,7 @@ module ScoutCabinet
     tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
     tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    database = CONNECTIONS[path] ||= tokyocabinet_class.new
+    database = Persist::CONNECTIONS[path] ||= tokyocabinet_class.new
 
     if big and not Open.exists?(path)
       database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
@@ -39,7 +39,7 @@ module ScoutCabinet
 
     database.open(path, tokyocabinet_class::OREADER)
 
-    CONNECTIONS[path] = database
+    Persist::CONNECTIONS[path] = database
 
     database
   end

data/lib/scout/tsv/persist.rb

@@ -4,8 +4,6 @@ require_relative 'persist/tokyocabinet'
 Persist.save_drivers[:tsv] = proc do |file,content| 
   stream = if IO === content
              content
-           elsif content.respond_to?(:get_stream)
-             content.get_stream
            elsif content.respond_to?(:stream)
              content.stream
            end

data/lib/scout/tsv/traverse.rb

@@ -1,48 +1,143 @@
 require_relative 'parser'
 module TSV
-  def self.traverse_add(into, res)
-    case into
-    when TSV::Dumper
-      into.add *res
-    when TSV, Hash
-      key, value = res
-      into[key] = value
-    end
+  def self.identify_field(key_field, fields, name)
+    return :key if name == :key || key_field.start_with?(name.to_s)
+    name.collect!{|n| key_field == n ? :key : n } if Array === name
+    NamedArray.identify_name(fields, name)
+  end
+
+  def identify_field(name)
+    TSV.identify_field(@key_field, @fields, name)
   end
 
-  def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
-    case obj
-    when TSV
-      self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
-    when String
-      f = Open.open(obj)
-      self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
-    when Step
-      self.traverse(obj.get_stream, into: into, cpus: cpus, bar: bar, **options, &block)
-    when IO
-      if into
-        into_thread = Thread.new do 
-          Thread.current.report_on_exception = false
-          Thread.current["name"] = "Traverse into"
-          TSV.parse obj, **options do |k,v|
-            begin
-              res = block.call k, v
-              traverse_add into, res
-            rescue
-              into.abort $!
+  def traverse(key_field_pos = :key, fields_pos = nil, type: nil, one2one: false, unnamed: false, key_field: nil, fields: nil, &block)
+    key_field = key_field_pos if key_field.nil?
+    fields = fields_pos if fields.nil?
+    type = @type if type.nil?
+    key_pos = self.identify_field(key_field)
+    fields = [fields] unless fields.nil? || Array === fields
+    positions = fields.nil? ? nil : self.identify_field(fields)
+
+    if key_pos == :key
+      key_name = @key_field
+    else
+      key_name = @fields[key_pos]
+      if positions.nil?
+        positions = (0..@fields.length-1).to_a
+        positions.delete_at key_pos
+        positions.unshift :key
+      end
+    end
+
+    if positions.nil? && key_pos == :key
+      field_names = @fields
+    elsif positions.nil? && key_pos != :key
+      field_names = @fields.dup
+      field_names.delete_at key_pos
+    elsif positions.include?(:key)
+      field_names = positions.collect{|p| p == :key ? @key_field : @fields[p] }
+    else
+      field_names = @fields.values_at *positions
+    end
+
+    key_index = positions.index :key if positions
+    positions.delete :key if positions
+
+    each do |key,values|
+      values = [values] if @type == :single
+      if positions.nil?
+        if key_pos != :key
+          values = values.dup
+          key = values.delete_at(key_pos)
+        end
+      else 
+        orig_key = key
+        key = values[key_pos] if key_pos != :key 
+
+        values = values.values_at(*positions)
+        if key_index
+          if @type == :double
+            values.insert key_index, [orig_key]
+          else
+            values.insert key_index, orig_key
+          end
+        end
+      end
+
+      if Array === key 
+        if @type == :double && one2one
+          if one2one == :fill
+            key.each_with_index do |key_i,i|
+              if type == :double
+                v_i = values.collect{|v| [v[i] || v.first] }
+              else
+                v_i = values.collect{|v| v[i] || v.first }
+              end
+              yield key_i, v_i
+            end
+          else
+            key.each_with_index do |key_i,i|
+              if type == :double
+                v_i = values.collect{|v| [v[i]] }
+              else
+                v_i = values.collect{|v| v[i] }
+              end
+              yield key_i, v_i
+            end
+          end
+        else
+          key.each_with_index do |key_i, i|
+            if type == :double
+              yield key_i, values
+            elsif type == :list
+              yield key_i, values.collect{|v| v[i] }
+            elsif type == :flat
+              yield key_i, values.flatten
+            elsif type == :single
+              yield key_i, values.first
             end
-            nil
           end
-          into.close if into.respond_to?(:close)
         end
-        Thread.pass until into_thread
-        into
       else
-        TSV.parse obj, **options do |k,v|
-          block.call k, v
-          nil
+        if type == @type
+          if type == :single
+            yield key, values.first
+          else
+            yield key, values
+          end
+        else
+          case [type, @type]
+          when [:double, :list]
+            yield key, values.collect{|v| [v] }
+          when [:double, :flat]
+            yield key, [values]
+          when [:double, :single]
+            yield key, [values]
+          when [:list, :double]
+            yield key, values.collect{|v| v.first }
+          when [:list, :flat]
+            yield key, [values.first]
+          when [:list, :single]
+            yield key, values
+          when [:flat, :double]
+            yield key, values.flatten
+          when [:flat, :list]
+            yield key, values.flatten
+          when [:flat, :single]
+            yield key, values
+          when [:single, :double]
+            yield key, values.flatten.first
+          when [:single, :list]
+            yield key, values.first
+          when [:single, :flat]
+            yield key, values.first
+          end
         end
       end
     end
+
+    [key_name, field_names]
   end
+
+  alias through traverse
 end