scout-gear 10.4.0 → 10.6.1

data/lib/scout/persist/engine/packed_index.rb

@@ -0,0 +1,100 @@
+class PackedIndex
+  attr_accessor :persistence_path, :mask, :mask_length, :offset, :item_size, :stream, :nil_string
+
+  ELEMS = {
+    "i" => ["l", 4],
+    "I" => ["q", 8],
+    "f" => ["f", 4],
+    "F" => ["d", 8],
+  }
+
+  def self.process_mask(mask)
+    str = ""
+    size = 0
+    mask.each do |e|
+      if ELEMS.include? e
+        str << ELEMS[e][0]
+        size += ELEMS[e][1]
+      elsif e =~ /^(\d+)s$/
+        num = $1.to_i
+        str << "a" << num.to_s
+        size += num
+      else
+        e, num = e.split(":")
+        str << e
+        size = (num.nil? ? size + 1 : size + num.to_i)
+      end
+    end
+    [str, size]
+  end
+
+  def size
+    @size ||= begin
+                (File.size(persistence_path) - offset) / item_size
+              end
+  end
+
+  def initialize(persistence_path, write = false, pattern = nil)
+    @persistence_path = persistence_path
+    if write
+      @stream = Open.open(persistence_path, :mode => 'wb')
+      @mask, @item_size = PackedIndex.process_mask pattern
+      header = [@mask.length, @item_size].pack("ll")
+      @stream.write(header)
+      @stream.write(mask)
+      @offset = @mask.length + 8
+    else
+      @stream = Open.open(persistence_path, :mode => 'rb')
+      header = @stream.read(8)
+      mask_length, @item_size = header.unpack("ll")
+      @mask = @stream.read(mask_length)
+      @offset = @mask.length + 8
+    end
+    @nil_string = "NIL" << ("-" * (@item_size - 3))
+  end
+
+  def file
+    @persistence_path
+  end
+  
+  def close
+    @stream.close
+  end
+
+  def write(force = false)
+    close
+    @stream = Open.open(persistence_path, :mode => 'wb')
+  end
+
+  def read(force = false)
+    close
+    @stream = Open.open(persistence_path, :mode => 'rb')
+  end
+
+  def <<(payload)
+    if payload.nil?
+      @stream.write nil_string
+    else
+      @stream.write payload.pack(mask)
+    end
+  end
+
+  def get_position(position)
+    @stream.seek(position * item_size + offset)
+    encoded = @stream.read(item_size)
+    return nil if encoded.nil? or encoded == nil_string 
+    encoded.unpack mask
+  end
+
+  alias [] get_position
+
+  def values_at(*positions)
+    positions.collect{|p|
+      get_position[p]
+    }
+  end
+
+  def close
+    @stream.close unless @stream.closed?
+  end
+end

data/lib/scout/persist/engine/sharder.rb

@@ -0,0 +1,219 @@
+class Sharder
+  attr_accessor :persistence_path, :shard_function, :databases, :closed, :writable, :mutex, :db_type, :persist_options
+
+  def initialize(persistence_path, write=false, db_type=nil, persist_options={}, &block)
+    @shard_function = persist_options[:shard_function] || block
+    @persist_options = persist_options
+    @persistence_path = Path.setup(persistence_path)
+    @mutex = Mutex.new
+    @writable = write
+    @db_type = db_type
+
+    if write
+      @databases = {} 
+    end
+  end
+
+  def persistence_path=(path)
+    @persistence_path = path
+    databases.values.each{|db| db.persistence_path = File.join(path, File.basename(db.persistence_path))}
+  end
+
+  def databases
+    @databases ||= begin
+                     hash = {}
+                     @persistence_path.glob('shard-*').each do |f|
+                       next if f.end_with?('.metadata')
+                       shard = File.basename(f).match(/shard-(.*)/)[1]
+                       hash[shard] = Persist.open_database(f, false, :clean, db_type, @persist_options)
+                     end
+                     hash
+                   end
+  end
+
+  def database(key)
+    shard = shard_function.call(key)
+    if databases.include? shard
+      databases[shard]
+    else
+      database = databases[shard] ||= begin
+                                        path = File.join(persistence_path, 'shard-' << shard.to_s)
+                                        (writable or File.exist?(path)) ? Persist.open_database(path, (File.exist?(path) ? false : writable), :clean, db_type, @persist_options) : nil
+                                      end
+      Log.warn "Database #{ path } missing" if database.nil?
+      database
+    end
+  end
+
+  MAX_CHAR = 255.chr
+
+  def prefix(key)
+    range(key, 1, key + MAX_CHAR, 1)
+  end
+
+  def get_prefix(key)
+    keys = prefix(key)
+    select(:key => keys)
+  end
+
+  def closed?
+    @closed
+  end
+
+  def close
+    @closed = true
+    super
+  end
+
+  def read(force = false)
+    raise "SIOT"
+    return if not write? and not closed and not force
+    self.close
+    databases.each{|d| d.read }
+    @writable = false
+    @closed = false
+    self
+  end
+
+  def write(force = true)
+    return if write? and not closed and not force
+    self.close
+
+    databases.each{|d| d.write }
+
+    @writable = true
+    @closed = false
+    self
+  end
+
+  def write?
+    @writable
+  end
+
+  def read?
+    ! write?
+  end
+
+  def range(*args)
+    databases.values.inject([]) do |acc,database|
+      acc.concat database.range(*args) if TokyoCabinet::BDB === database
+      acc
+    end
+  end
+
+  def each
+    databases.values.each do |database|
+      database.each do |k,v|
+        yield k, v
+      end
+    end
+  end
+
+  def include?(key)
+    self[key] != nil
+  end
+
+  def collect
+    res = []
+    each do |key, value|
+      res << if block_given?
+               yield key, value
+      else
+        [key, value]
+      end
+    end
+    res
+  end
+
+  def write_and_read
+    lock_filename = Persist.persistence_path(File.join(persistence_path, 'write'), {:dir => TSV.lock_dir})
+    Misc.lock(lock_filename) do
+      @mutex.synchronize do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                read
+              end
+        res
+      end
+    end
+  end
+
+  def write_and_close
+    lock_filename = Persist.persistence_path(File.join(persistence_path, 'write'), {:dir => TSV.lock_dir})
+    Misc.lock(lock_filename) do
+      @mutex.synchronize do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+  end
+
+  def read_and_close
+    @mutex.synchronize do
+      read if @closed or not read?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+  end
+
+  def merge!(hash)
+    hash.each do |key,values|
+      self[key] = values
+    end
+  end
+
+  def keys
+    databases.values.collect{|d| d.keys }
+  end
+
+  def []=(key, value, clean = false)
+    database(key).send(:[]=, key, value)
+  end
+
+  def [](key, clean=false)
+    database = database(key)
+    return nil if database.nil?
+    database.send(:[], key)
+  end
+
+  #def <<(p)
+  #  return if p.nil?
+  #  self[p.first] = p.last
+  #end
+
+  def <<(values)
+    key, rest = values
+    database(key) << values
+  end
+
+  def write
+    databases.values.each{|database| database.write }
+  end
+
+  def read(force = false)
+    databases.values.each{|database| database.read(force) }
+  end
+
+  def close
+    databases.values.each{|database| database.close }
+  end
+
+  def size
+    databases.inject(0){|acc,i| 
+      shard, db = i; 
+      acc += db.size 
+    }
+  end
+end
+

data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb

@@ -1,5 +1,4 @@
 require 'tkrzw'
-require_relative 'adapter'
 
 module ScoutTKRZW
   attr_accessor :persistence_path, :persistence_class, :open_options
@@ -66,19 +65,3 @@ module ScoutTKRZW
     search("contain", "")
   end
 end
-
-Persist.save_drivers[:tkh] = proc do |file, content|
-  data = ScoutTKRZW.open(file, true, "tkh")
-  content.annotate(data)
-  data.extend TSVAdapter
-  data.merge!(content)
-  data.close
-  data.read
-  data
-end
-
-Persist.load_drivers[:tkh] = proc do |file| 
-  data = ScoutTKRZW.open(file, false, "tkh")
-  data.extend TSVAdapter unless TSVAdapter === data
-  data
-end

data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb

@@ -1,10 +1,9 @@
 require 'tokyocabinet'
-require_relative 'adapter'
 
 module ScoutCabinet
   attr_accessor :persistence_path, :persistence_class
 
-  def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+  def self.open(path, write = true, tokyocabinet_class = TokyoCabinet::HDB)
     path = path.find if Path === path
     if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
       big = true
@@ -25,7 +24,7 @@ module ScoutCabinet
     database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
     if big and not Open.exists?(path)
-      database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+      database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
     end
 
     flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
@@ -56,7 +55,7 @@ module ScoutCabinet
   end
 
   def read(force = false)
-    return if ! write? && ! closed && ! force
+    return if ! @writable && ! @closed && ! force
     self.close
     if !self.open(@persistence_path, persistence_class::OREADER)
       ecode = self.ecode
@@ -69,8 +68,17 @@ module ScoutCabinet
     self
   end
 
+  def write?
+    @writable
+  end
+
+  def closed?
+    @closed
+  end
+
+
   def write(force = true)
-    return if write? && ! closed && ! force
+    return if write? && ! closed? && ! force
     self.close
 
     if !self.open(@persistence_path, persistence_class::OWRITER)
@@ -84,35 +92,51 @@ module ScoutCabinet
     self
   end
 
-  #def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
-  #  raise
-  #  write = true unless File.exist? path
-
-  #  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+  def write_and_read
+    begin
+      write
+      yield
+    ensure
+      read
+    end
+  end
 
-  #  database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+  def write_and_close
+    begin
+      write
+      yield
+    ensure
+      close
+    end
+  end
 
-  #  unless serializer == :clean
-  #    TSV.setup database
-  #    database.write_and_read do
-  #      database.serializer = serializer
-  #    end if serializer && database.serializer != serializer
-  #  end
+  def self.importtsv(database, stream)
+    begin
+      bin = case database
+            when TokyoCabinet::HDB
+              'tchmgr'
+            when TokyoCabinet::BDB
+              'tcbmgr'
+            else
+              raise "Database not HDB or BDB: #{Log.fingerprint database}"
+            end
+      
+      database.close
+      CMD.cmd("#{bin} version", :log => false)
+      FileUtils.mkdir_p File.dirname(database.persistence_path)
+      CMD.cmd("#{bin} importtsv '#{database.persistence_path}'", :in => stream, :log => false, :dont_close_in => true)
+    rescue
+      Log.debug("tchmgr importtsv failed for: #{database.persistence_path}")
+    end
+  end
 
-  #  database
-  #end
-end
+  class << self
+    alias load_stream importtsv
+  end
 
-Persist.save_drivers[:HDB] = proc do |file, content|
-  data = ScoutCabinet.open(file, true, "HDB")
-  content.annotate(data)
-  data.extend TSVAdapter
-  data.merge!(content)
-  data
-end
+  def importtsv(stream)
+    ScoutCabinet.load_stream(self, stream)
+  end
 
-Persist.load_drivers[:HDB] = proc do |file| 
-  data = ScoutCabinet.open(file, false, "HDB")
-  data.extend TSVAdapter unless TSVAdapter === data
-  data
+  alias load_stream importtsv
 end

data/lib/scout/persist/engine.rb

@@ -0,0 +1,4 @@
+require_relative 'engine/tokyocabinet'
+require_relative 'engine/fix_width_table'
+require_relative 'engine/packed_index'
+require_relative 'engine/sharder'

data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb}

@@ -1,6 +1,5 @@
 require 'scout/open/lock'
-require 'scout/meta_extension'
-require_relative 'serialize'
+require 'scout/annotation'
 
 module TSVAdapter
   attr_accessor :persistence_path, :persistence_class, :closed, :writable, :serializer
@@ -12,81 +11,114 @@ module TSVAdapter
     end
   end
 
-  EXTENSION_ATTR_HASH_KEY = "__extension_attr_hash__"
-  EXTENSION_ATTR_HASH_SERIALIZER = Marshal
+  ANNOTATION_ATTR_HASH_KEY = "__annotation_hash__"
+  ANNOTATION_ATTR_HASH_SERIALIZER = Marshal
 
   def serializer=(serializer)
     @serializer = Symbol === serializer ? SERIALIZER_ALIAS[serializer] : serializer
   end
 
-  def load_extension_attr_hash
-    EXTENSION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(EXTENSION_ATTR_HASH_KEY)))
+  def load_annotation_hash
+    ANNOTATION_ATTR_HASH_SERIALIZER.load(StringIO.new(self.orig_get(ANNOTATION_ATTR_HASH_KEY)))
   end
 
-  def save_extension_attr_hash
-    self.orig_set(EXTENSION_ATTR_HASH_KEY, EXTENSION_ATTR_HASH_SERIALIZER.dump(self.extension_attr_hash))
+  def save_annotation_hash
+    self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
   end
 
   def self.extended(base)
-    return if base.respond_to?(:orig_set)
-
-    class << base
-      alias orig_set []=
-        alias orig_get []
-
-      def [](key)
-        self.read_lock do
-          load_value(super(key))
-        end
+    if TSV === base
+      base.instance_variable_get(:@annotations).push(:serializer)
+      base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
+      base.save_annotation_hash
+    else
+      begin
+        TSV.setup(base, base.load_annotation_hash)
+      rescue
+        TSV.setup(base)
+        base.save_annotation_hash
       end
+    end
+  end
 
-      def []=(key, value)
-        self.write_lock do
-          super(key, save_value(value))
-        end
-      end
+  def []=(...) super(...); end
+  def [](...) super(...); end
+  def keys(...) super(...); end
+  def each(...) super(...); end
+  def size(...) super(...); end
 
-      def load_value(str)
-        return nil if str.nil?
-        serializer.load(str)
-      end
+  alias orig_set []=
+  alias orig_get []
+  alias orig_keys keys
+  alias orig_each each
+  alias orig_size size
 
-      def save_value(value)
-        serializer.dump(value)
-      end
+  def [](key, clean = false)
+    self.read_lock do
+      v = super(key)
+      return v if clean
+      v = load_value(v)
+      NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
+      v
     end
+  end
 
-    if base.include?(EXTENSION_ATTR_HASH_KEY)
-      TSV.setup(base, base.load_extension_attr_hash)
-    else
-      base.instance_variable_get(:@extension_attrs).push :serializer
-      base.serializer = SERIALIZER_ALIAS[base.type] if base.serializer.nil?
-      base.save_extension_attr_hash
+  def []=(key, value, clean = false)
+    self.write_lock do
+      if clean
+        super(key, value)
+      else
+        super(key, save_value(value))
+      end
     end
   end
 
-  def keys(*args)
+  def load_value(str)
+    return nil if str.nil?
+    return str if serializer.nil?
+    return load_value(str.first) if Array === str
+    serializer.load(str)
+  end
+
+  def save_value(value)
+    return nil if value.nil?
+    return value if serializer.nil?
+    serializer.dump(value)
+  end
+
+  def keys_annotation_hash_key(*args)
     k = self.read_lock do
-      super(*args)
+      orig_keys(*args)
     end
 
-    if k[0] == EXTENSION_ATTR_HASH_KEY
+    if k[0] == ANNOTATION_ATTR_HASH_KEY
       k.slice(1,k.length)
-    elsif k[-1] == EXTENSION_ATTR_HASH_KEY
+    elsif k[-1] == ANNOTATION_ATTR_HASH_KEY
       k.slice(0,k.length-1)
     else
-      k - [EXTENSION_ATTR_HASH_KEY]
+      k - [ANNOTATION_ATTR_HASH_KEY]
     end
   end
+  alias keys keys_annotation_hash_key
 
-  def each(&block)
+  def each_annotation_hash_key(&block)
     self.read_lock do
-      super do |k,v|
-        next if k == EXTENSION_ATTR_HASH_KEY
+      orig_each do |k,v|
+        next if k == ANNOTATION_ATTR_HASH_KEY
         yield(k, load_value(v))
       end
     end
   end
+  alias each each_annotation_hash_key
+
+  def size_annotation_hash_key(*args)
+    self.read_lock do
+      orig_size(*args) - 1 # Discount the ANNOTATION_ATTR_HASH_KEY
+    end
+  end
+  alias size size_annotation_hash_key
+
+  alias length size
 
   def collect(&block)
     res = []
@@ -122,7 +154,10 @@ module TSVAdapter
 
   def write(*args)
     begin
-      super(*args)
+      begin
+        super(*args) 
+      rescue
+      end
       @writable = true
     rescue NoMethodError
     end
@@ -321,12 +356,6 @@ module TSVAdapter
     select(:key => keys)
   end
 
-  def size(*args)
-    self.read_lock do
-      super(*args)
-    end
-  end
-
   def values_at(*keys)
     self.read_lock do
       keys.collect do |k|