rbbt-util 5.5.68 → 5.6.0

data/lib/rbbt/persist/tsv/kyotocabinet.rb

@@ -0,0 +1,168 @@
+require 'kyotocabinet'
+
+module Persist
+
+  module KCAdapter
+    attr_accessor :persistence_path, :kyotocabinet_class, :closed, :writable
+
+    def self.open(path, write, kyotocabinet_class = "kch")
+      real_path = path + ".#{kyotocabinet_class}"
+
+      @persistence_path = real_path
+
+      flags = (write ? KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE : nil)
+      database = 
+        CONNECTIONS[path] ||= begin
+                                db = KyotoCabinet::DB.new
+                                db.open(real_path, flags)
+                                db
+                              end
+
+      database.extend KCAdapter
+      database.persistence_path ||= real_path
+
+      database
+    end
+
+    def keys
+      keys = []
+      each_key{|k| keys.concat k}
+      keys
+    end
+
+    def prefix(key)
+      range(key, 1, key + 255.chr, 1)
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+    def include?(key)
+      value = get(key)
+      ! value.nil?
+    end
+
+    def closed?
+      @closed
+    end
+
+    def close
+      @closed = true
+      super
+    end
+
+    def read(force = false)
+      return if not write? and not closed and not force
+      self.close
+      if !self.open(@persistence_path, KyotoCabinet::DB::OREADER)
+        raise "Open error #{ res }. Trying to open file #{@persistence_path}"
+      end
+      @writable = false
+      @closed = false
+      self
+    end
+
+    def write(force = true)
+      return if write? and not closed and not force
+      self.close
+
+      if !self.open(@persistence_path, KyotoCabinet::DB::OWRITER)
+        raise "Open error. Trying to open file #{@persistence_path}"
+      end
+
+      @writable = true
+      @closed = false
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def collect
+      res = []
+      each do |key, value|
+        res << if block_given?
+                 yield key, value
+        else
+          [key, value]
+        end
+      end
+      res
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                read
+              end
+        res
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def read_and_close
+      read if @closed or write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+    #def []=(key,value)
+    #  super(key,value)
+    #  self.synchronize
+    #end
+
+
+    def range(*args)
+      super(*args) - TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+
+  def self.open_kyotocabinet(path, write, serializer = nil,  kyotocabinet_class= 'kch')
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::KCAdapter.open(path, write, kyotocabinet_class)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end

data/lib/rbbt/persist/tsv/leveldb.rb

@@ -0,0 +1,121 @@
+require 'leveldb'
+
+module Persist
+
+  module LevelDBAdapter
+    attr_accessor :persistence_path, :closed, :writable
+
+    def self.open(path, write)
+
+      database = CONNECTIONS[path] ||= begin
+                                         LevelDB::DB.new path
+                                       end
+
+      database.extend Persist::LevelDBAdapter unless Persist::LevelDBAdapter === database
+      database.persistence_path ||= path
+      ddd database
+
+      database
+    end
+
+    def prefix(key)
+      range(key, 1, key + 255.chr, 1)
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+    def include?(key)
+      includes?(key)
+    end
+
+    def closed?
+      @closed
+    end
+
+    def close
+      @closed = true
+    end
+
+    def read(force = false)
+      self
+    end
+
+    def write(force = true)
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                read
+              end
+        res
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def read_and_close
+      read if @closed or write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+
+    def range(*args)
+      super(*args) - TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+
+  def self.open_leveldb(path, write, serializer = nil)
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::LevelDBAdapter.open(path, write)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end

data/lib/rbbt/persist/tsv/lmdb.rb

@@ -0,0 +1,148 @@
+require 'lmdb'
+
+module Persist
+
+  module LMDBAdapter
+    attr_accessor :persistence_path, :closed
+
+    def self.open(path, write)
+
+      database = CONNECTIONS[path] ||= begin
+                                         dir = File.dirname(File.expand_path(path))
+                                         file = File.basename(path)
+                                         env = LMDB.new(dir, :mapsize => 1024 * 10000)
+                                         database = env.database file, :create => write
+                                         database
+                                       end
+
+      database.extend Persist::LMDBAdapter unless Persist::LMDBAdapter === database
+      database.persistence_path ||= path
+
+      database
+    end
+
+    def keys
+      keys = []
+      cursor do |cursor|
+        while p = cursor.next
+          keys << p.first
+        end
+      end
+      keys
+    end
+    
+    def include?(key)
+      self.send(:[], key, true)
+    end
+
+    def closed?
+      false
+    end
+
+    def close
+      self
+    end
+
+    def read(force = false)
+      self
+    end
+
+    def write(force = true)
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def each
+      cursor do |cursor|
+        while pair = cursor.next
+          yield *pair
+        end
+      end
+      self
+    end
+
+    def collect
+      res = []
+      cursor do |cursor|
+        while pair = cursor.next
+          res = if block_given?
+                  yield *pair
+                else
+                  pair
+                end
+        end
+      end
+      res
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                read
+              end
+        res
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def read_and_close
+      read if @closed or write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+
+    def range(*args)
+      super(*args) - TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+
+  def self.open_lmdb(path, write, serializer = nil)
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::LMDBAdapter.open(path, write)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -0,0 +1,158 @@
+require 'tokyocabinet'
+
+module Persist
+
+  module TCAdapter
+    attr_accessor :persistence_path, :tokyocabinet_class, :closed, :writable
+
+    def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB"
+      tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+
+      database = CONNECTIONS[path] ||= tokyocabinet_class.new
+
+      flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+      database.close
+
+      if !database.open(path, flags)
+        ecode = database.ecode
+        raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+      end
+
+      database.extend Persist::TCAdapter unless Persist::TCAdapter === database
+      database.persistence_path ||= path
+      database.tokyocabinet_class = tokyocabinet_class
+
+      database
+    end
+
+    MAX_CHAR = 255.chr
+
+    def prefix(key)
+      range(key, 1, key + MAX_CHAR, 1)
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+    def closed?
+      @closed
+    end
+
+    def close
+      @closed = true
+      super
+    end
+
+    def read(force = false)
+      return if not write? and not closed and not force
+      self.close
+      if !self.open(@persistence_path, tokyocabinet_class::OREADER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
+      @writable = false
+      @closed = false
+      self
+    end
+
+    def write(force = true)
+      return if write? and not closed and not force
+      self.close
+
+      if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
+
+      @writable = true
+      @closed = false
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def collect
+      res = []
+      each do |key, value|
+        res << if block_given?
+                 yield key, value
+        else
+          [key, value]
+        end
+      end
+      res
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                read
+              end
+        res
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def read_and_close
+      read if @closed or write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+
+    def range(*args)
+      super(*args) #- TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+
+  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end