rbbt-util 5.13.23 → 5.13.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 716be8fb2d3ec4a9e9cc76482f96771856976191
-  data.tar.gz: 958406db01b705360855b2a6bff88b0e0ff804b0
+  metadata.gz: 4cb02f1584dee7188f5a2bc771f733eb81c46f1a
+  data.tar.gz: 47850a9f40b69fc8cb761dbefea3aec895c1c955
 SHA512:
-  metadata.gz: a3e9739aba63e67c6507ac8e7b31393d6eaae60dde59e5c0bea00faf3132b732a9f5248cb52e3d552c57834cd157258c2799f9993c2caff35d0a65f2c723548b
-  data.tar.gz: 22aa1b9d063dbe6d6150e47e9949dd863ed0c7ba8cdaa90fbb1b61ea4a588ed696694a25daf5f1dfabf54e130339083e55c684a7e78ff37625596f92a8643199
+  metadata.gz: f3459787e3d4aa401917919e6a70bbb4c314477dbce5f9251f0723dbc905af70170f0e844b6d905fe9fd86a232acd1cc98553d3ed84ebca91743a06f0149610c
+  data.tar.gz: 2028a7f0dc8de1adfb4a69e54f139d4a2a329c6e094b4043618792314874e7ee4da08b72327912800d899cca258cccbf015dee6f10f13ffabb9283132f823929

data/lib/rbbt/persist/tsv.rb

@@ -28,8 +28,6 @@ rescue Exception
   Log.debug "The kyotocabinet gem could not be loaded. Persistance using this engine will fail."
 end
 
-require 'rbbt/persist/tsv/sharder'
-
 module Persist
   CONNECTIONS = {}
 
@@ -85,14 +83,23 @@ module Persist
 
     if is_persisted?(path) and not persist_options[:update]
       Log.debug "TSV persistence up-to-date: #{ path }"
-      return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+      if persist_options[:shard_function]
+        return open_sharder(path, false, nil, persist_options[:engine], &persist_options[:shard_function]) 
+      else
+        return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+      end
     end
 
     Misc.lock lock_filename do
       begin
         if is_persisted?(path) and not persist_options[:update]
           Log.debug "TSV persistence (suddenly) up-to-date: #{ path }"
-          return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+
+          if persist_options[:shard_function]
+            return open_sharder(path, false, nil, persist_options[:engine], &persist_options[:shard_function]) 
+          else
+            return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB) 
+          end
         end
 
         FileUtils.rm path if File.exists? path
@@ -101,8 +108,15 @@ module Persist
 
         tmp_path = path + '.persist'
 
-        data = open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
-        data.serializer = :type if TSV === data and data.serializer.nil?
+        data = if persist_options[:shard_function]
+                 open_sharder(tmp_path, true, persist_options[:serializer], persist_options[:engine], &persist_options[:shard_function]) 
+               else
+                 open_database(tmp_path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB) 
+               end
+
+        if TSV === data and data.serializer.nil?
+          data.serializer = :type 
+        end
 
         data.write_and_read do
           yield data
@@ -123,3 +137,6 @@ module Persist
     end
   end
 end
+
+require 'rbbt/persist/tsv/sharder'
+

data/lib/rbbt/persist/tsv/sharder.rb

@@ -1,23 +1,66 @@
-require 'rbbt-util'
-
 module Persist
+  module SharderAdapter
+    def self.open(path, write, type=nil, &block)
+
+      database = CONNECTIONS[path] ||= Sharder.new(path, write, type, &block)
+
+      database.extend Persist::SharderAdapter unless Persist::SharderAdapter === database
+
+      database
+    end
+
+  end
+
   class Sharder
-    attr_accessor :directory, :params, :shard_function, :databases, :closed, :writable, :mutex
+    attr_accessor :persistence_path, :shard_function, :databases, :closed, :writable, :mutex, :db_type
 
-    def initialize(directory, *rest, &block)
+    def initialize(persistence_path, write = false, db_type=nil, &block)
       @shard_function = block
-      @params = rest
-      @databases = {}
-      @directory = directory
+      @persistence_path = Path.setup(persistence_path)
       @mutex = Mutex.new
+      @writable = write
+      @db_type = db_type
+
+      if write
+        @databases = {} 
+      end
+    end
+
+    def <<(key,value)
+      self[key] = value
+    end
+
+    def persistence_path=(path)
+      @persistence_path = path
+    end
+
+    def databases
+      @databases ||= begin
+                       hash = {}
+                       @persistence_path.glob('shard-*').each do |f|
+                         shard = File.basename(f).match(/shard-(.*)/)[1]
+                         hash[shard] = Persist.open_database(f, false, :clean, db_type)
+                       end
+                       hash
+                     end
     end
 
     def database(key)
-      shard = shard_function.call(key)
-      databases[shard] ||= begin
-                             path = File.join(directory, 'shard-' << shard.to_s)
-                             Persist.open_database(path, *params)
-                           end
+      shard = key =~ /__tsv_/ ? "0" : shard_function.call(key)
+      if databases.include? shard
+        databases[shard]
+      else
+        database ||= begin
+                       path = File.join(persistence_path, 'shard-' << shard.to_s)
+                       (writable or File.exists?(path)) ? Persist.open_database(path, writable, :clean, db_type) : nil
+                     end
+        if database
+          databases[shard] = database 
+        else
+          Log.warn "Database #{ path } missing" if
+          nil
+        end
+      end
     end
 
     MAX_CHAR = 255.chr
@@ -69,13 +112,17 @@ module Persist
     end
 
     def each
-      databases.each do |database|
+      databases.values.each do |database|
         database.each do |k,v|
           yield k, v
         end
       end
     end
 
+    def include?(key)
+      self[key] != nil
+    end
+
     def collect
       res = []
       each do |key, value|
@@ -89,7 +136,7 @@ module Persist
     end
 
     def write_and_read
-      lock_filename = Persist.persistence_path(File.join(directory, 'write'), {:dir => TSV.lock_dir})
+      lock_filename = Persist.persistence_path(File.join(persistence_path, 'write'), {:dir => TSV.lock_dir})
       Misc.lock(lock_filename) do
         @mutex.synchronize do
           write if @closed or not write?
@@ -104,7 +151,7 @@ module Persist
     end
 
     def write_and_close
-      lock_filename = Persist.persistence_path(File.join(directory, 'write'), {:dir => TSV.lock_dir})
+      lock_filename = Persist.persistence_path(File.join(persistence_path, 'write'), {:dir => TSV.lock_dir})
       Misc.lock(lock_filename) do
         @mutex.synchronize do
           write if @closed or not write?
@@ -137,15 +184,15 @@ module Persist
     end
 
     def keys
-      databases.values.collect{|d| d.keys }.flatten
+      databases.values.collect{|d| d.keys }.flatten - TSV::ENTRY_KEYS.to_a
     end
 
-    def []=(key, value)
-      database(key)[key] = value
+    def []=(key, value, clean = false)
+      database(key).send(:[]=, key, value)
     end
 
-    def [](key, value)
-      database(key)[key]
+    def [](key, clean=false)
+      v = database(key).send(:[], key)
     end
 
     def <<(p)
@@ -165,4 +212,19 @@ module Persist
       databases.values.each{|database| database.close }
     end
   end
+
+  def self.open_sharder(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB, &shard_function)
+    write = true unless File.exists? path
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    database = Persist::SharderAdapter.open(path, write, tokyocabinet_class, &shard_function)
+
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer if serializer
+    end
+
+    database
+  end
 end

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -5,8 +5,8 @@ module Persist
   module TCAdapter
     attr_accessor :persistence_path, :tokyocabinet_class, :closed, :writable, :mutex
 
-    def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
-      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB"
+    def self.open(path, write, serializer, tokyocabinet_class = TokyoCabinet::HDB)
+      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
       database = CONNECTIONS[path] ||= tokyocabinet_class.new
@@ -80,12 +80,6 @@ module Persist
     def read?
       ! write?
     end
-    #def each
-    #  iterinit
-    #  while key = iternext
-    #    yield key, get(key)
-    #  end
-    #end
 
     def collect
       res = []
@@ -164,7 +158,7 @@ module Persist
 
     FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
 
-    database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+    database = Persist::TCAdapter.open(path, write, serializer, tokyocabinet_class)
 
     unless serializer == :clean
       TSV.setup database

data/lib/rbbt/tsv/accessor.rb

@@ -243,7 +243,7 @@ module TSV
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer == serializer_module
+      value = serializer_module.load(value) unless value.nil? or serializer_module.nil? or TSV::CleanSerializer == serializer_module
 
       # Annotated with Entity and NamedArray
       if not @unnamed
@@ -417,13 +417,15 @@ module TSV
   end
 
   def namespace=(value)
-    self.send(:[]=, "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
+    self.send(:[]=, "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : TSV::TSV_SERIALIZER.dump(value), true)
     @namespace = value
     @entity_options = nil
   end
 
   def fields=(value)
-    self.send(:[]=, "__tsv_hash_fields", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
+    clean = true
+    value_ym = value.nil? ? SERIALIZED_NIL : TSV::TSV_SERIALIZER.dump(value)
+    self.send(:[]=, "__tsv_hash_fields", value_ym, clean)
     @fields = value
     @named_fields = nil
   end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -54,7 +54,7 @@ module TSV
   def self.report(msg, obj, into)
     into = into[:into] if Hash === into and into.include? :into
 
-    Log.medium "#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"
+    Log.medium{"#{ msg } #{stream_name(obj)} -> #{stream_name(into)}"}
   end
 
   #{{{ TRAVERSE OBJECTS
@@ -135,7 +135,7 @@ module TSV
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
     if File === io and io.closed? 
       begin
-        Log.medium "Rewinding stream #{stream_name(io)}"
+        Log.medium{"Rewinding stream #{stream_name(io)}"}
         io.reopen io.filename, "r"
       rescue
         Log.exception $!
@@ -168,7 +168,7 @@ module TSV
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
     if File === io and io.closed? 
       begin
-        Log.medium "Rewinding stream #{stream_name(io)}"
+        Log.medium{"Rewinding stream #{stream_name(io)}"}
         io.reopen io.filename, "r"
       rescue
         Log.exception $!
@@ -192,7 +192,7 @@ module TSV
       options[:type] = :single
     end
 
-    Log.medium "Traversing #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
+    Log.medium{"Traversing #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
     begin
       case obj
       when TSV
@@ -256,28 +256,28 @@ module TSV
         raise "Unknown object for traversal: #{Misc.fingerprint obj }"
       end
     rescue IOError
-      Log.medium "IOError traversing #{stream_name(obj)}: #{$!.message}"
+      Log.medium{"IOError traversing #{stream_name(obj)}: #{$!.message}"}
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
       raise $!
     rescue Errno::EPIPE
-      Log.medium "Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"
+      Log.medium{"Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"}
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
       raise $!
     rescue Aborted
-      Log.medium "Aborted traversing #{stream_name(obj)}"
+      Log.medium{"Aborted traversing #{stream_name(obj)}"}
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
-      Log.medium "Aborted traversing 2 #{stream_name(obj)}"
+      Log.medium{"Aborted traversing 2 #{stream_name(obj)}"}
     rescue Exception
-      Log.medium "Exception traversing #{stream_name(obj)}"
+      Log.medium{"Exception traversing #{stream_name(obj)}"}
       Log.exception $!
       stream = obj_stream(obj)
       stream.abort if stream and stream.respond_to? :abort
@@ -335,7 +335,7 @@ module TSV
         q.clean
       end
     rescue Interrupt, Aborted
-      Log.medium "Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"
+      Log.medium{"Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"}
       q.abort
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort

data/test/rbbt/persist/test_tsv.rb

@@ -24,13 +24,13 @@ class TestPersistTSV < Test::Unit::TestCase
       Misc.benchmark(1, "Build database with #{MAX - 2} entries") do
         db = TSV.open(file, :fields => [1], :persist => true, :persist_engine => engine, :persist_dir => tmp_file, :type => :single, :unnamed => true)
       end
-      test = db.keys.sort{rand}[1..100000]
+      _test = db.keys.sort{rand}[1..100000]
       Misc.benchmark(5, "Access #{test.length} random entries") do
-        test.each do |k| db[k] end
+        _test.each do |k| db[k] end
       end
       Log.info "Profiling access to #{test.length} random entries"
       Misc.profile :min_percent => 0.1 do
-        test.each do |k| db[k] end
+        _test.each do |k| db[k] end
       end
       assert_equal "1:10611:G", db["rs189107123"]
     end

data/test/rbbt/persist/tsv/test_sharder.rb

@@ -1,26 +1,38 @@
 require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
 require 'test/unit'
+require 'rbbt-util'
 require 'rbbt/persist/tsv'
 
 class TestSharder < Test::Unit::TestCase
   def test_shard
     TmpFile.with_file do |dir|
-      sharder = Persist::Sharder.new dir, true, :float_array, 'HDB' do |key|
+      shard_function = Proc.new do |key|
         key[-1]
       end
 
-      keys = []
-      size = 1_000_000
-      Misc.benchmark(2) do
-        sharder.write_and_read do
-          size.times do |v| 
-            keys << v.to_s
-            sharder[v.to_s] = [v, v*2]
-          end
+      size = 10
+      sharder = Persist.persist_tsv(nil, "ShardTest", {}, :file => dir, :shard_function => shard_function, :persist => true, :serializer => :float_array) do |db|
+        size.times do |v| 
+          db[v.to_s] = [v, v*2]
         end
+        db
+      end
+      assert_equal dir, sharder.persistence_path
+      assert_equal size, sharder.keys.length
+
+      assert_equal [2,4], sharder["2"]
+      count = 0
+      sharder.through do |k,v|
+        count += 1
+      end
+      assert_equal count, size
 
-        assert_equal size, sharder.keys.length
+      sharder = Persist::Sharder.new dir do |key|
+        key[-1]
       end
+
+      assert_equal size, sharder.keys.length
+
     end
   end
 end

data/test/rbbt/test_tsv.rb

@@ -519,4 +519,12 @@ row2    A AA AAA
     end
  
   end
+
+  def test_shard
+    shard_function = Proc.new do |key|
+      key[-1]
+    end
+    tsv = datafile_test('identifiers').tsv :persist => true, :shard_function => shard_function
+    assert_equal 10000, tsv.keys.length + 2
+  end
 end

data/test/test_helper.rb

@@ -28,6 +28,6 @@ class Test::Unit::TestCase
   end
 
   def datafile_test(file)
-    File.join(File.dirname(__FILE__), 'data', file)
+    Path.setup(File.join(File.dirname(__FILE__), 'data', file))
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.23
+  version: 5.13.24
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake