scout-gear 7.2.0 → 8.0.0

data/lib/scout/tsv/persist/adapter.rb

@@ -1,7 +1,9 @@
 require_relative '../../open/lock'
+require_relative '../../meta_extension'
+require_relative 'serialize'
 
 module TSVAdapter
-  attr_accessor :persistence_path, :persistence_class, :closed, :writable
+  attr_accessor :persistence_path, :persistence_class, :closed, :writable, :serializer
 
   class << self
     attr_accessor :lock_dir
@@ -13,20 +15,20 @@ module TSVAdapter
   EXTENSION_ATTR_HASH_KEY = "__extension_attr_hash__"
   EXTENSION_ATTR_HASH_SERIALIZER = Marshal
 
+  def serializer=(serializer)
+    @serializer = Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
+  end
+
   def load_extension_attr_hash
-    EXTENSION_ATTR_HASH_SERIALIZER.load(self[EXTENSION_ATTR_HASH_KEY])
+    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(EXTENSION_ATTR_HASH_KEY)))
   end
 
   def save_extension_attr_hash
-    self[EXTENSION_ATTR_HASH_KEY]= EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash)
+    self.orig_set(EXTENSION_ATTR_HASH_KEY, EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash))
   end
 
   def self.extended(base)
-    if base.include?(EXTENSION_ATTR_HASH_KEY)
-      TSV.setup(base, base.load_extension_attr_hash)
-    elsif TSV === base
-      base[EXTENSION_ATTR_HASH_KEY] = EXTENSION_ATTR_HASH_SERIALIZER.dump(base.extension_attr_hash)
-    end
+    return if base.respond_to?(:orig_set)
 
     class << base
       alias orig_set []=
@@ -43,37 +45,24 @@ module TSVAdapter
           super(key, save_value(value))
         end
       end
-    end
 
-    case base.type
-    when :single
-      class << base
-        def load_value(value)
-          value
-        end
-        def save_value(value)
-          value
-        end
+      def load_value(str)
+        return nil if str.nil?
+        serializer.load(str)
       end
-    when :list, :flat
-      class << base
-        def load_value(value)
-          value.nil? ? nil : value.split("\t")
-        end
-        def save_value(value)
-          value * "\t"
-        end
-      end
-    when :double
-      class << base
-        def load_value(value)
-          value.nil? ? nil : value.split("\t").collect{|v| v.split("|") }
-        end
-        def save_value(value)
-          value.collect{|v| v * "|" } * "\t"
-        end
+
+      def save_value(value)
+        serializer.dump(value)
       end
     end
+
+    if base.include?(EXTENSION_ATTR_HASH_KEY)
+      TSV.setup(base, base.load_extension_attr_hash)
+    else
+      base.instance_variable_get(:@extension_attrs).push :serializer
+      base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
+      base.save_extension_attr_hash
+    end
   end
 
   def keys(*args)
@@ -82,9 +71,9 @@ module TSVAdapter
     end
 
     if k[0] == EXTENSION_ATTR_HASH_KEY
-      k.slice(1,k.length-1)
+      k.slice(1,k.length)
     elsif k[-1] == EXTENSION_ATTR_HASH_KEY
-      k.slice(0,k.length-2)
+      k.slice(0,k.length-1)
     else
       k - [EXTENSION_ATTR_HASH_KEY]
     end

data/lib/scout/tsv/persist/fix_width_table.rb

@@ -0,0 +1,327 @@
+class FixWidthTable
+
+  attr_accessor :filename, :file, :value_size, :record_size, :range, :size, :mask, :write
+  def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
+    filename = filename.find if Path === filename
+    @filename = filename
+
+    if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
+      Log.debug "FixWidthTable create: #{ filename }"
+      @value_size  = value_size
+      @range       = range
+      @record_size = @value_size + (@range ? 16 : 8)
+      @write = true
+
+      if %w(memory stringio).include?(filename.to_s.downcase)
+        @filename = :memory
+        @file     = StringIO.new
+      else
+        FileUtils.rm @filename if File.exist? @filename
+        FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
+        @file = File.open(@filename, 'w:ASCII-8BIT')
+      end
+
+      @file.write [value_size].pack("L")
+      @file.write [@range ? 1 : 0 ].pack("C")
+
+      @size = 0
+    else
+      Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
+      if in_memory
+        @file        = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
+      else
+        @file        = File.open(@filename, 'r:ASCII-8BIT')
+      end
+      @value_size  = @file.read(4).unpack("L").first
+      @range       = @file.read(1).unpack("C").first == 1
+      @record_size = @value_size + (@range ? 16 : 8)
+      @write = false
+
+      @size        = (File.size(@filename) - 5) / (@record_size)
+    end
+
+    @mask = "a#{@value_size}"
+  end
+
+  def write?
+    @write
+  end
+
+  def persistence_path
+    @filename
+  end
+
+  def persistence_path=(value)
+    @filename=value
+  end
+
+  def self.get(filename, value_size = nil, range = nil, update = false)
+    return self.new(filename, value_size, range, update) if filename == :memory
+    case
+    when (!File.exist?(filename) or update or not Persist::CONNECTIONS.include?(filename))
+      Persist::CONNECTIONS[filename] = self.new(filename, value_size, range, update)
+    end
+
+    Persist::CONNECTIONS[filename] 
+  end
+
+  def format(pos, value)
+    padding = value_size - value.length
+    if @range
+      (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
+    else
+      [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
+    end
+  end
+
+  def add(pos, value)
+    format = format(pos, value)
+    @file.write format
+
+    @size += 1
+  end
+
+  def last_pos
+    pos(size - 1)
+  end
+
+  def idx_pos(index)
+    return nil if index < 0 or index >= size
+    @file.seek(5 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_pos_end(index)
+    return nil if index < 0 or index >= size
+    @file.seek(9 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_overlap(index)
+    return nil if index < 0 or index >= size
+    @file.seek(13 + (record_size) * index, IO::SEEK_SET)
+    @file.read(4).unpack("l").first
+  end
+
+  def idx_value(index)
+    return nil if index < 0 or index >= size
+    @file.seek((@range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    padding = @file.read(4).unpack("l").first+1
+    txt = @file.read(value_size)
+    str = txt.unpack(mask).first
+    padding > 1 ? str[0..-padding] : str
+  end
+
+  def read(force = false)
+    return if @filename == :memory
+    @write = false
+    @file.close unless @file.closed?
+    @file = File.open(filename, 'r:ASCII-8BIT')
+  end
+
+  def close
+    @write = false
+    @file.close
+  end
+
+  def dump
+    read
+    @file.rewind
+    @file.read
+  end
+
+  #{{{ Adding data
+
+  def add_point(data)
+    data.sort_by{|value,pos| pos }.each do |value, pos|
+      add pos, value
+    end
+  end
+
+  def add_range_point(pos, value)
+    @latest ||= []
+    while @latest.any? and @latest[0] < pos[0]
+      @latest.shift
+    end
+    overlap = @latest.length
+    add pos + [overlap], value
+    @latest << pos[1]
+  end
+
+  def add_range(data)
+    @latest = []
+    data.sort_by{|value, pos| pos[0] }.each do |value, pos|
+      add_range_point(pos, value)
+    end
+  end
+
+  #{{{ Searching
+
+  def closest(pos)
+    upper = size - 1
+    lower = 0
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = lower + (upper - lower) / 2
+      pos_idx = idx_pos(idx)
+
+      case pos <=> pos_idx
+      when 0
+        break
+      when -1
+        upper = idx - 1
+      when 1
+        lower = idx + 1
+      end
+    end
+
+    if pos_idx > pos
+      idx = idx - 1
+    end
+
+    idx.to_i
+  end
+
+  def get_range(pos, return_idx = false)
+    case pos
+    when Range
+      r_start = pos.begin
+      r_end   = pos.end
+    when Array
+      r_start, r_end = pos
+    else
+      r_start, r_end = pos, pos
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx < 0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    overlap = idx_overlap(idx)
+
+    idx -= overlap unless overlap.nil?
+
+    values = []
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    
+    if return_idx
+      while l_start <= r_end
+        values << idx if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx) if l_end >= r_start 
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    end
+
+    values
+  end
+
+  def get_point(pos, return_idx = false)
+    if Range === pos
+      r_start = pos.begin
+      r_end   = pos.end
+    else
+      r_start = pos.to_i
+      r_end   = pos.to_i
+    end
+
+    idx = closest(r_start)
+
+    return [] if idx >= size
+    return [] if idx < 0 and r_start == r_end
+
+    idx = 0 if idx < 0
+
+    idx += 1 unless idx_pos(idx) >= r_start
+
+    return [] if idx >= size
+
+    values = []
+    l_start = idx_pos(idx)
+    l_end   = idx_pos_end(idx)
+    if return_idx 
+      while l_start <= r_end
+        values << idx
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    else
+      while l_start <= r_end
+        values << idx_value(idx)
+        idx += 1
+        break if idx >= size
+        l_start = idx_pos(idx)
+        l_end   = idx_pos_end(idx)
+      end
+    end
+
+    values
+  end
+
+  def [](pos)
+    return [] if size == 0
+    self.read
+    if @range
+      get_range(pos)
+    else
+      get_point(pos)
+    end
+  end
+  
+  def overlaps(pos, value = false)
+    return [] if size == 0
+    idxs = if @range
+      get_range(pos, true)
+    else
+      get_point(pos, true)
+    end
+    if value
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx), idx_value(idx)] * ":"}
+    else
+      idxs.collect{|idx| [idx_pos(idx), idx_pos_end(idx)] * ":"}
+    end
+  end
+
+
+  def values_at(*list)
+    list.collect{|pos|
+      self[pos]
+    }
+  end
+
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
+
+  alias length size
+end
+
+Persist.save_drivers[:fwt] = proc do |file, content|
+  content.file.seek 0
+  Misc.sensiblewrite(file, content.file.read)
+end
+
+Persist.load_drivers[:fwt] = proc do |file| 
+  FixWidthTable.new file
+end

data/lib/scout/tsv/persist/serialize.rb

@@ -0,0 +1,117 @@
+require 'base64'
+module TSVAdapter
+
+  class CleanSerializer
+    def self.dump(o); o end
+    def self.load(o); o end
+  end
+
+  class BinarySerializer
+    def self.dump(o); [o].pack('m'); end
+    def self.load(str); str.unpack('m').first; end
+  end
+  
+  class IntegerSerializer
+    def self.dump(i); [i].pack("l"); end
+    def self.load(str); str.unpack("l").first; end
+  end
+
+  class FloatSerializer
+    def self.dump(i); [i].pack("d"); end
+    def self.load(str); str.unpack("d").first; end
+  end
+
+  class StrictIntegerArraySerializer
+    def self.dump(a); a.pack("l*"); end
+    def self.load(str); a = str.unpack("l*"); end
+  end
+
+  class StrictFloatArraySerializer
+    def self.dump(a); a.pack("d*"); end
+    def self.load(str); a = str.unpack("d*"); end
+  end
+
+  class IntegerArraySerializer
+    NIL_INT = -999
+    def self.dump(a); a.collect{|v| v || NIL_INT}.pack("l*"); end
+    def self.load(str); a = str.unpack("l*"); a.collect{|v| v == NIL_INT ? nil : v}; end
+  end
+
+  class FloatArraySerializer
+    NIL_FLOAT = -999.999
+    def self.dump(a); a.collect{|v| v || NIL_FLOAT}.pack("d*"); end
+    def self.load(str); a = str.unpack("d*"); a.collect{|v| v == NIL_FLOAT ? nil : v}; end
+  end
+
+  class StringSerializer
+    def self.dump(str); str.to_s; end
+    def self.load(str); str.dup; end
+  end
+
+  class StringArraySerializer
+    def self.dump(array)
+      array.collect{|a| a.to_s} * "\t"
+    end
+
+    def self.load(string)
+      return nil if string.nil? or string == 'nil'
+      return [] if string.empty?
+      string.split("\t", -1)
+    end
+  end
+
+  class StringDoubleArraySerializer
+    def self.dump(array)
+      begin
+      array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
+      rescue Encoding::CompatibilityError
+        array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
+      end
+    end
+
+    def self.load(string)
+      return [] if string.nil?
+      string.split("\t", -1).collect{|l| l.split("|", -1)}
+    end
+  end
+
+  class TSVMarshalSerializer
+    def self.dump(tsv)
+      Marshal.dump(tsv.dup)
+    end
+
+    def self.load(string)
+      TSV.setup Marshal.load(string)
+    end
+  end
+
+  class TSVSerializer
+    def self.dump(tsv)
+      tsv.to_s
+    end
+
+    def self.load(string)
+      TSV.open StringIO.new(string)
+    end
+  end
+
+  SERIALIZER_ALIAS = {
+    :single => StringSerializer,
+    :list => StringArraySerializer,
+    :flat => StringArraySerializer,
+    :double => StringDoubleArraySerializer,
+    :clean => CleanSerializer,
+    :integer => IntegerSerializer, 
+    :float => FloatSerializer, 
+    :integer_array => IntegerArraySerializer,
+    :float_array => FloatArraySerializer,
+    :strict_integer_array => StrictIntegerArraySerializer,
+    :strict_float_array => StrictFloatArraySerializer,
+    :marshal => Marshal,
+    :string => StringSerializer,
+    :binary => BinarySerializer,
+    :tsv => TSVSerializer,
+    :marshal_tsv => TSVMarshalSerializer
+  }
+
+end

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -4,8 +4,8 @@ require_relative 'adapter'
 module ScoutCabinet
   attr_accessor :persistence_path, :persistence_class
 
-  CONNECTIONS = {}
   def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+    path = path.find if Path === path
     if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
       big = true
       tokyocabinet_class = tokyocabinet_class.split(":").first
@@ -16,10 +16,13 @@ module ScoutCabinet
     dir = File.dirname(File.expand_path(path))
     Open.mkdir(dir) unless File.exist?(dir)
 
+    tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
     tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
     tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    database = CONNECTIONS[path] ||= tokyocabinet_class.new
+    # Hack - Ignore warning: undefining the allocator of T_DATA class
+    # TokyoCabinet::HDB_data
+    database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
     if big and not Open.exists?(path)
       database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
@@ -39,7 +42,7 @@ module ScoutCabinet
 
     database.open(path, tokyocabinet_class::OREADER)
 
-    CONNECTIONS[path] = database
+    Persist::CONNECTIONS[path] = database
 
     database
   end

data/lib/scout/tsv/persist.rb

@@ -4,10 +4,12 @@ require_relative 'persist/tokyocabinet'
 Persist.save_drivers[:tsv] = proc do |file,content| 
   stream = if IO === content
              content
-           elsif content.respond_to?(:get_stream)
-             content.get_stream
            elsif content.respond_to?(:stream)
              content.stream
+           elsif content.respond_to?(:dumper_stream)
+             content.dumper_stream
+           else
+             content
            end
   Open.sensible_write(file, stream)
 end