rbbt-util 5.13.22 → 5.13.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a19d56208ed15434da77c5823b332526972ac70
-  data.tar.gz: 0d1296b74615471612be8717e6530c1cdf687147
+  metadata.gz: 716be8fb2d3ec4a9e9cc76482f96771856976191
+  data.tar.gz: 958406db01b705360855b2a6bff88b0e0ff804b0
 SHA512:
-  metadata.gz: 2a409449bb5de7f586d78b4b99306a94fef4a4613da789a06175586e4c16800c6094e797fc2caeab057b9ea2c45ddba173a8bd79e0f341e80733fd8fa515f6d6
-  data.tar.gz: 5da483b8a7179b5fd9e9f9b49c1eed1024a7d3d78af15df4cc28f6af66472f8572e4de0840ca2b2bbca24f624abf1d8002e019330d42b304cea03f4cf1543481
+  metadata.gz: a3e9739aba63e67c6507ac8e7b31393d6eaae60dde59e5c0bea00faf3132b732a9f5248cb52e3d552c57834cd157258c2799f9993c2caff35d0a65f2c723548b
+  data.tar.gz: 22aa1b9d063dbe6d6150e47e9949dd863ed0c7ba8cdaa90fbb1b61ea4a588ed696694a25daf5f1dfabf54e130339083e55c684a7e78ff37625596f92a8643199

data/lib/rbbt/persist/tsv/sharder.rb

@@ -0,0 +1,168 @@
+require 'rbbt-util'
+
+module Persist
+  class Sharder
+    attr_accessor :directory, :params, :shard_function, :databases, :closed, :writable, :mutex
+
+    def initialize(directory, *rest, &block)
+      @shard_function = block
+      @params = rest
+      @databases = {}
+      @directory = directory
+      @mutex = Mutex.new
+    end
+
+    def database(key)
+      shard = shard_function.call(key)
+      databases[shard] ||= begin
+                             path = File.join(directory, 'shard-' << shard.to_s)
+                             Persist.open_database(path, *params)
+                           end
+    end
+
+    MAX_CHAR = 255.chr
+
+    def prefix(key)
+      range(key, 1, key + MAX_CHAR, 1)
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+    def closed?
+      @closed
+    end
+
+    def close
+      @closed = true
+      super
+    end
+
+    def read(force = false)
+      return if not write? and not closed and not force
+      self.close
+      databases.each{|d| d.read }
+      @writable = false
+      @closed = false
+      self
+    end
+
+    def write(force = true)
+      return if write? and not closed and not force
+      self.close
+
+      databases.each{|d| d.write }
+
+      @writable = true
+      @closed = false
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def read?
+      ! write?
+    end
+
+    def each
+      databases.each do |database|
+        database.each do |k,v|
+          yield k, v
+        end
+      end
+    end
+
+    def collect
+      res = []
+      each do |key, value|
+        res << if block_given?
+                 yield key, value
+        else
+          [key, value]
+        end
+      end
+      res
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(File.join(directory, 'write'), {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        @mutex.synchronize do
+          write if @closed or not write?
+          res = begin
+                  yield
+                ensure
+                  read
+                end
+          res
+        end
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(File.join(directory, 'write'), {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        @mutex.synchronize do
+          write if @closed or not write?
+          res = begin
+                  yield
+                ensure
+                  close
+                end
+          res
+        end
+      end
+    end
+
+    def read_and_close
+      @mutex.synchronize do
+        read if @closed or not read?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+    def keys
+      databases.values.collect{|d| d.keys }.flatten
+    end
+
+    def []=(key, value)
+      database(key)[key] = value
+    end
+
+    def [](key, value)
+      database(key)[key]
+    end
+
+    def <<(p)
+      return if p.nil?
+      self[p.first] = p.last
+    end
+
+    def write
+      databases.values.each{|database| database.write }
+    end
+
+    def read
+      databases.values.each{|database| database.read }
+    end
+
+    def close
+      databases.values.each{|database| database.close }
+    end
+  end
+end

data/lib/rbbt/persist/tsv.rb

@@ -28,6 +28,8 @@ rescue Exception
   Log.debug "The kyotocabinet gem could not be loaded. Persistance using this engine will fail."
 end
 
+require 'rbbt/persist/tsv/sharder'
+
 module Persist
   CONNECTIONS = {}
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -1,4 +1,7 @@
 module TSV
+
+  module MultipleResult; end
+
   def self.obj_stream(obj)
     case obj
     when (defined? Step and Step)
@@ -15,7 +18,7 @@ module TSV
   def self.guess_max(obj)
     begin
       case obj
-      when Step
+      when (defined? Step and Step)
         if obj.done?
           CMD.cmd("wc -l '#{obj.path.find}'").read.to_i
         else
@@ -27,8 +30,15 @@ module TSV
         CMD.cmd("wc -l '#{obj.filename}'").read.to_i
       when Path
         CMD.cmd("wc -l '#{obj.find}'").read.to_i
+      when String
+        if File.exists? obj
+          CMD.cmd("wc -l '#{obj}'").read.to_i
+        else
+          nil
+        end
       end
     rescue Exception
+      Log.exception $!
       nil
     end
   end
@@ -69,6 +79,7 @@ module TSV
         end
       end
     end
+    Log::ProgressBar.remove_bar(bar) if bar
     join.call if join
   end
 
@@ -92,6 +103,7 @@ module TSV
         end
       end
     end
+    Log::ProgressBar.remove_bar(bar) if bar
     join.call if join
   end
 
@@ -115,6 +127,7 @@ module TSV
         end
       end
     end
+    Log::ProgressBar.remove_bar(bar) if bar
     join.call if join
   end
 
@@ -132,14 +145,22 @@ module TSV
 
     if callback
       while line = io.gets
-        callback.call yield line.strip
-        bar.tick if bar
+        begin
+          callback.call yield line.strip
+        ensure
+          bar.tick if bar
+        end
       end
     else
       while line = io.gets
-        yield line.strip
+        begin
+          yield line.strip
+        ensure
+          bar.tick if bar
+        end
       end
     end
+    Log::ProgressBar.remove_bar(bar) if bar
     join.call if join
   end
 
@@ -332,6 +353,12 @@ module TSV
   end
 
   def self.store_into(store, value)
+    if MultipleResult === value
+      value.each do |v|
+        store_into store, v
+      end
+      return
+    end
     begin
       case store
       when Hash

data/test/rbbt/persist/tsv/test_sharder.rb

@@ -0,0 +1,27 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'test/unit'
+require 'rbbt/persist/tsv'
+
+class TestSharder < Test::Unit::TestCase
+  def test_shard
+    TmpFile.with_file do |dir|
+      sharder = Persist::Sharder.new dir, true, :float_array, 'HDB' do |key|
+        key[-1]
+      end
+
+      keys = []
+      size = 1_000_000
+      Misc.benchmark(2) do
+        sharder.write_and_read do
+          size.times do |v| 
+            keys << v.to_s
+            sharder[v.to_s] = [v, v*2]
+          end
+        end
+
+        assert_equal size, sharder.keys.length
+      end
+    end
+  end
+end
+

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -366,4 +366,14 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
     assert_equal size, stream.read.split("\n").length
   end
+
+  def test_store_multiple
+    size = 1000
+    array = (1..size).to_a.collect{|n| n.to_s}
+    stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
+      sleep 0.01
+      [e,e+".alt"].extend TSV::MultipleResult
+    end
+    assert_equal size*2, stream.read.split("\n").length
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.22
+  version: 5.13.23
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-27 00:00:00.000000000 Z
+date: 2014-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -136,6 +136,7 @@ files:
 - lib/rbbt/persist/tsv/kyotocabinet.rb
 - lib/rbbt/persist/tsv/leveldb.rb
 - lib/rbbt/persist/tsv/lmdb.rb
+- lib/rbbt/persist/tsv/sharder.rb
 - lib/rbbt/persist/tsv/tokyocabinet.rb
 - lib/rbbt/resource.rb
 - lib/rbbt/resource/path.rb
@@ -278,6 +279,7 @@ files:
 - test/rbbt/persist/tsv/test_kyotocabinet.rb
 - test/rbbt/persist/tsv/test_leveldb.rb
 - test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
 - test/rbbt/persist/tsv/test_tokyocabinet.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/test_annotations.rb
@@ -395,6 +397,7 @@ test_files:
 - test/rbbt/persist/test_tsv.rb
 - test/rbbt/persist/tsv/test_lmdb.rb
 - test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_sharder.rb
 - test/rbbt/persist/tsv/test_cdb.rb
 - test/rbbt/persist/tsv/test_tokyocabinet.rb
 - test/rbbt/persist/tsv/test_leveldb.rb