rbbt-util 5.5.68 → 5.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3d30a17283c3778cfd2cd645c141e964f5cbd96
-  data.tar.gz: d656ccf4110c9286694893e9378eaeb555301d53
+  metadata.gz: b772991027bae39d5d75e7d19fdfc628b2a78871
+  data.tar.gz: aed0bd56fbd0178bff6cd37b386390a392a5a7ec
 SHA512:
-  metadata.gz: 488d486e35e367e10271d1fe11339dd4c9027e5290b0290b1ca4f9a6f4ea9c0735df68c09a988363526a77aa7c21aeda49e0ccaa5ead8011b2ea9c447db820a2
-  data.tar.gz: d343c02b1236aee587ad6258898b487d9b5c24d17f444746fb4df2a94d12e6cc3b85b3542a6387029768ed9f9385fc7557dd6538131cdca321a2324664993b05
+  metadata.gz: 6aeb956836ee4d77b643249dc63c13ba6aed3aa9568b58173eb872c45e384d5202ea8e742c55e775187371f4e5b438198d57f56896e3daad9041158320692677
+  data.tar.gz: eab7947a295e70759b50bf6333c589f7c71aa535d5c5c26f0c848aacb5b9f723505799bb4baec4c16f30b5fe3f9a5ed8715f3f6326612b33b342823622428497

data/lib/rbbt/association.rb

@@ -242,7 +242,7 @@ module Association
       tsv = load_tsv(file, options)
 
       tsv.annotate(data)
-      data.serializer = tsv.type if TokyoCabinet::HDB === data
+      data.serializer = tsv.type if data.respond_to? :persistence_path
 
       tsv.with_unnamed do
         tsv.each do |k,v|

data/lib/rbbt/association/index.rb

@@ -11,6 +11,7 @@ module Association
       repo.extend Association::Index
       repo.parse_key_field
       repo.unnamed = true
+      repo
     end
 
     def reverse
@@ -56,7 +57,7 @@ module Association
         acc
       end
     end
- 
+
     #{{{ Subset
 
     def select_entities(entities)

data/lib/rbbt/entity.rb

@@ -117,6 +117,8 @@ module Entity
               Hash === res ? res[self] : res[0]
             end
           end
+        else 
+          raise "Type not undestood in property: #{ type }"
 
         end
       end

data/lib/rbbt/persist/tsv.rb

@@ -1,237 +1,35 @@
 begin
-  require 'tokyocabinet'
+  require 'rbbt/persist/tsv/tokyocabinet'
 rescue Exception
   Log.warn "The tokyocabinet gem could not be loaded: persistence over TSV files will fail"
 end
 
-module Persist
-  TC_CONNECTIONS = {}
-
-  def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
-    write = true unless File.exists? path
-
-    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB"
-    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
-
-    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-
-    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
-
-    database = TC_CONNECTIONS[path] ||= tokyocabinet_class.new
-    database.close
-
-    if !database.open(path, flags)
-      ecode = database.ecode
-      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
-    end
-
-    if not database.respond_to? :old_close
-      class << database
-        attr_accessor :writable, :closed, :persistence_path, :tokyocabinet_class
-
-        def prefix(key)
-          range(key, 1, key + 255.chr, 1)
-        end
-
-        def get_prefix(key)
-          keys = prefix(key)
-          select(:key => keys)
-        end
-
-        def closed?
-          @closed
-        end
-
-        alias old_close close
-        def close
-          @closed = true
-          old_close
-        end
-
-        def read(force = false)
-          return if not write? and not closed and not force
-          self.close
-          if !self.open(@persistence_path, tokyocabinet_class::OREADER)
-            ecode = self.ecode
-            raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-          end
-          @writable = false
-          @closed = false
-          self
-        end
-
-        def write(force = true)
-          return if write? and not closed and not force
-          self.close
-
-          if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
-            ecode = self.ecode
-            raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-          end
-
-          @writable = true
-          @closed = false
-          self
-        end
-
-        def write?
-          @writable
-        end
-
-        def collect
-          res = []
-          each do |key, value|
-            res << if block_given?
-                     yield key, value
-            else
-              [key, value]
-            end
-          end
-          res
-        end
-
-        def delete(key)
-          out(key)
-        end
-
-        def write_and_read
-          lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-          Misc.lock(lock_filename) do
-            write if @closed or not write?
-            res = begin
-                    yield
-                  ensure
-                    read
-                  end
-            res
-          end
-        end
-
-        def write_and_close
-          lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-          Misc.lock(lock_filename) do
-            write if @closed or not write?
-            res = begin
-                    yield
-                  ensure
-                    close
-                  end
-            res
-          end
-        end
-
-        def read_and_close
-          read if @closed or write?
-          res = begin
-                  yield
-                ensure
-                  close
-                end
-          res
-        end
-
-        def merge!(hash)
-          hash.each do |key,values|
-            self[key] = values
-          end
-        end
-
-        if instance_methods.include? "range"
-          alias old_range range
-
-          def range(*args)
-            keys = old_range(*args)
-            keys - TSV::ENTRY_KEYS
-          end
-        end
-      end
-    end
-
-    database.persistence_path ||= path
-    database.tokyocabinet_class = tokyocabinet_class
-
-    unless serializer == :clean
-      TSV.setup database
-      database.serializer = serializer || database.serializer
-      database.fields
-    end
-
-    database
-  end
-
-  def self.persist_tsv(source, filename, options = {}, persist_options = {})
-    persist_options[:prefix] ||= "TSV"
-
-    data = case
-           when persist_options[:data]
-             persist_options[:data]
-           when persist_options[:persist]
-
-             filename ||= case
-                          when Path === source
-                            source
-                          when (source.respond_to?(:filename) and source.filename)
-                            source.filename
-                          when source.respond_to?(:cmd)
-                            "CMD-#{Misc.digest(source.cmd)}"
-                          when TSV === source
-                            "TSV[#{Misc.digest Misc.fingerprint(source)}]"
-                          else
-                            source.object_id.to_s
-                          end
-
-             filename ||= source.object_id.to_s
-
-             path = persistence_path(filename, persist_options, options)
-
-             if is_persisted? path and not persist_options[:update]
-               Log.debug "TSV persistence up-to-date: #{ path }"
-               lock_filename = Persist.persistence_path(path, {:dir => TSV.lock_dir})
-               return Misc.lock(lock_filename) do open_tokyocabinet(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB); end
-             else
-               Log.medium "TSV persistence creating: #{ path }"
-             end
-
-             FileUtils.rm path if File.exists? path
-
-             tmp_path = path + '.persist'
-
-             data = open_tokyocabinet(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
-             data.serializer = :type if TSV === data and data.serializer.nil?
-
-             data.close
-
-             data
-           else
-             {}
-           end
+begin
+  require 'rbbt/persist/tsv/lmdb'
+rescue Exception
+  Log.warn "The lmdb gem could not be loaded. Persistance using this engine will fail."
+end
 
-    begin
-      if data.respond_to? :persistence_path and data != persist_options[:data]
-        data.write_and_close do
-          yield data
-        end
-      else
-        yield data
-      end
-    rescue Exception
-      Log.error "Captured error during persist_tsv. Erasing: #{path}"
-      FileUtils.rm tmp_path if tmp_path and File.exists? tmp_path
-      raise $!
-    ensure
-      data.close if data.respond_to? :close
-      if tmp_path 
-        FileUtils.mv tmp_path, path if File.exists? tmp_path and not File.exists? path
-        tsv = TC_CONNECTIONS[path] = TC_CONNECTIONS.delete tmp_path
-        tsv.persistence_path = path
-      end
-    end
+begin
+  require 'rbbt/persist/tsv/leveldb'
+rescue Exception
+  Log.warn "The LevelDB gem could not be loaded. Persistance using this engine will fail."
+end
 
-    data.read if data.respond_to? :read and ((data.respond_to?(:write?) and data.write?) or (data.respond_to?(:closed?) and data.closed?))
+begin
+  require 'rbbt/persist/tsv/cdb'
+rescue Exception
+  Log.warn "The CDB gem could not be loaded. Persistance using this engine will fail."
+end
 
+begin
+  require 'rbbt/persist/tsv/kyotocabinet'
+rescue Exception
+  Log.warn "The kyotocabinet gem could not be loaded. Persistance using this engine will fail."
+end
 
-    data
-  end
+module Persist
+  CONNECTIONS = {}
 
   def self.get_filename(source)
     case
@@ -246,6 +44,21 @@ module Persist
     end || source.object_id.to_s
   end
 
+  def self.open_database(path, write, serializer = nil, type = "HDB")
+    case type
+    when "LevelDB"
+      Persist.open_leveldb(path, write, serializer)
+    when "CDB"
+      Persist.open_cdb(path, write, serializer)
+    when "LMDB"
+      Persist.open_lmdb(path, write, serializer)
+    when 'kch', 'kct'
+      Persist.open_kyotocabinet(path, write, serializer, type)
+    else
+      Persist.open_tokyocabinet(path, write, serializer, type)
+    end
+  end
+
   def self.persist_tsv(source, filename, options = {}, persist_options = {}, &block)
     persist_options[:prefix] ||= "TSV"
 
@@ -270,14 +83,14 @@ module Persist
 
     if is_persisted? path and not persist_options[:update]
       Log.debug "TSV persistence up-to-date: #{ path }"
-      return open_tokyocabinet(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+      return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
     end
 
     Misc.lock lock_filename do
       begin
         if is_persisted? path 
           Log.debug "TSV persistence up-to-date: #{ path }"
-          return open_tokyocabinet(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+          return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
         end
 
         FileUtils.rm path if File.exists? path
@@ -286,22 +99,23 @@ module Persist
 
         tmp_path = path + '.persist'
 
-        data = open_tokyocabinet(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
+        data = open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
         data.serializer = :type if TSV === data and data.serializer.nil?
 
         data.write_and_read do
           yield data
         end
 
-        FileUtils.mv tmp_path, path if File.exists? tmp_path and not File.exists? path
-        tsv = TC_CONNECTIONS[path] = TC_CONNECTIONS.delete tmp_path
+        FileUtils.mv data.persistence_path, path if File.exists? data.persistence_path and not File.exists? path
+        tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
         tsv.persistence_path = path
+        tsv.fix_io if tsv.respond_to? :fix_io
 
         data
       rescue Exception
         Log.error "Captured error during persist_tsv. Erasing: #{path}"
-        FileUtils.rm tmp_path if tmp_path and File.exists? tmp_path
-        FileUtils.rm path if path and File.exists? path
+        FileUtils.rm_rf tmp_path if tmp_path and File.exists? tmp_path
+        FileUtils.rm_rf path if path and File.exists? path
         raise $!
       end
     end

data/lib/rbbt/persist/tsv/cdb.rb

@@ -0,0 +1,139 @@
+require 'libcdb'
+
+module Persist
+
+  module CDBAdapter
+    attr_accessor :persistence_path, :closed
+
+    def self.open(path, write)
+      write = true unless File.exists? path
+
+     database = CONNECTIONS[path] ||= begin
+                                         file = File.open(path, 'w')
+                                         LibCDB::CDB.new(file)
+                                       end
+
+      database.extend Persist::CDBAdapter unless Persist::CDBAdapter === database
+      database.persistence_path ||= path
+
+      database
+    end
+
+    def include?(k)
+      not write? and super(k) 
+    end
+
+    def [](*args)
+      write? ?  nil : super(*args)
+    end
+
+    def []=(k,v)
+      if write?
+        add(k,v)
+      end
+    end
+
+    def closed?
+      @closed
+    end
+
+    def fix_io
+      if instance_variable_get(:@io) != persistence_path
+        #close_read if read?
+        #close_write if write?
+        instance_variable_set(:@io, File.open(persistence_path))
+      end
+    end
+
+    def close
+      self.closed = true
+      begin
+        fix_io
+        super
+      rescue
+      end
+    end
+
+    def read(force = false)
+      self.closed = false
+      fix_io
+      open_read
+    end
+
+    def write(force = true)
+      self.closed = false
+      fix_io
+      open_write
+      self
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              rescue
+                Log.error $!.message
+                Log.debug $!.backtrace * "\n"
+              ensure
+                read  if write?
+              end
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        write if @closed or not write?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def read_and_close
+      read if @closed or write?
+      res = begin
+              yield
+            ensure
+              close
+            end
+      res
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+
+    def range(*args)
+      super(*args) - TSV::ENTRY_KEYS.to_a
+    end
+  end
+
+
+  def self.open_cdb(path, write, serializer = nil)
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::CDBAdapter.open(path, write)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+    end
+
+    database
+  end
+end