hbase-jruby 0.6.3-java → 0.6.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0654a082ed7102051b33a0d27f9b317870f59eb2
-  data.tar.gz: a050c546803000e36f4daadc511beff5c3b24b51
+  metadata.gz: f36f9ad6226b500c087ad5a8ffc9f294c738752e
+  data.tar.gz: 23ac26035d0054188538e115d899c191dd803776
 SHA512:
-  metadata.gz: a8bb69c1d2909ad7ddff0062087eedf78552dbe75fbfcb9d95f5ad4e3c0e6a28b77b1a0529e4426aed8fab6ac267568a7d085938933419bd1354b8dff5285871
-  data.tar.gz: b0b872f8559795f34b2b546f55e7386f3763e7b2a14d7de61a7909a8ed3de68e7f606534f37ae156bb3f823fbca482c29dd13412cfa28a9cfb62b0caa4c63762
+  metadata.gz: cb25e98c711400af4e3eda1f91efbc0ca2a145f0c213384ea7da2bb45d4b98f6511381dee30c0a606faf9c1f10948aff5996d6c447cfc0acb41d98d71cd9349c
+  data.tar.gz: 58a59bf73bf759718bd58063af461cee14ece10de17cd40634b81dbf26a6406fa3d8dc608a842fde314bb42a74a537c9965e3073edb368e12f34b6961e1445dc

data/CHANGELOG.md

@@ -1,6 +1,29 @@
 Changelog
 =========
 
+0.6.4
+-----
+
+### Performance improvement (experimental)
+
+`HBase::Table` instance can be set up to cache the interpretations of the
+column keys using thread-locals which can lead to 3-times faster Put
+generation in tight loops.
+
+```ruby
+table = hbase[:my_table, cache: true]
+# ...
+table.close
+```
+
+However, the option is off by default because of the following issues:
+
+1. You should not use it with the tables with unlimited number of columns.
+2. Caching does not keep track of the updates of the schema. If you change the
+   schema of an `HBase` object after you created an `HBase::Table` object with
+   the cache turned on from it, the object may hold stale information on the
+   types of the columns. `HBase::Table#close` method will clean up the cache.
+
 0.6.3
 -----
 

data/lib/hbase-jruby/hbase.rb

@@ -9,6 +9,7 @@ class HBase
   attr_reader :config, :schema
 
   include Admin
+  include HBase::Util
 
   # @overload HBase.log4j=(filename)
   #   Configure Log4j logging with the given file
@@ -99,7 +100,7 @@ class HBase
     end
   end
 
-  # Closes HTablePool and connection
+  # Closes the connection, and clean up thread-local cache
   # @return [nil]
   def close
     @mutex.synchronize do
@@ -117,6 +118,10 @@ class HBase
         end if use_table_pool?
       end
     end
+
+    thread_local.delete self
+
+    nil
   end
 
   # Returns whether if the connection is closed
@@ -128,25 +133,25 @@ class HBase
   # Returns the list of HBase::Table instances
   # @return [Array<HBase::Table>]
   def tables
-    check_closed
     table_names.map { |tn| table(tn) }
   end
 
   # Returns the list of table names
   # @return [Array<String>]
   def table_names
-    check_closed
     with_admin { |admin| admin.list_tables.map(&:name_as_string) }
   end
   alias list table_names
 
   # Creates an HBase::Table instance for the specified name
   # @param [#to_s] table_name The name of the table
+  # @param [Hash] opts Options
+  #   @option opts [Boolean] :cache Use thread-local cache (default: false)
   # @return [HBase::Table]
-  def table table_name
+  def table table_name, opts = {}
     check_closed
 
-    ht = HBase::Table.send :new, self, @config, table_name
+    ht = HBase::Table.send :new, self, @config, table_name, opts[:cache]
 
     if block_given?
       yield ht

data/lib/hbase-jruby/table.rb

@@ -21,23 +21,16 @@ class Table
     check_closed
 
     # [:hbase_jruby][HBase connection][Table name]
-    local_vars = Thread.current[:hbase_jruby] ||= {}
-    unless local_htables = local_vars[@hbase]
-      local_htables = local_vars[@hbase] = {}
-    end
-    local_htables[@name] ||= @hbase.send :get_htable, @name
+    thread_local(@hbase, @name_sym)[:htable] ||= @hbase.send(:get_htable, @name)
   end
 
-  # Return HTable instance back to the table pool.
+  # Clean up thread-locals
   # Generally this is not required unless you use unlimited number of threads
   # @return [nil]
   def close
-    check_closed
-
-    (t = Thread.current[:hbase_jruby]) &&
-    (t = t[@hbase]) &&
-    (t = t.delete @name) &&
-    t.close
+    hash = thread_local(@hbase).delete(@name_sym)
+    hash[:htable].close if hash && hash[:htable]
+    nil
   end
 
   # Returns whether if the connection is closed
@@ -320,13 +313,23 @@ class Table
     @hbase.schema.lookup_and_parse @name_sym, col, expect_cq
   end
 
+  # @private
+  module ThreadLocalCache
+    def lookup_and_parse col, expect_cq
+      thread_local(@hbase, @name_sym, :columns)[col] ||=
+          @hbase.schema.lookup_and_parse(@name_sym, col, expect_cq)
+    end
+  end
+
 private
-  def initialize hbase, config, name
+  def initialize hbase, config, name, cache
     @hbase    = hbase
     @config   = config
     @name     = name.to_s
     @name_sym = name.to_sym
     @mutation = Mutation.new(self)
+
+    extend ThreadLocalCache if cache
   end
 
   def check_closed

data/lib/hbase-jruby/table/mutation.rb

@@ -27,6 +27,8 @@ class Mutation
               put.add cf, cq, Util.to_typed_bytes(t, v)
             end unless v.nil?
           end
+        when String
+          put.add cf, cq, val.to_java_bytes
         else
           put.add cf, cq, Util.to_typed_bytes(type, val)
         end

data/lib/hbase-jruby/util.rb

@@ -194,6 +194,16 @@ private
       raise ArgumentError, "Invalid time format"
     end
   end
+
+  # XXX Why am I doing this instead of using inject (reduce)?
+  # Because inject is around 25% slower, and this method is used in tight loops
+  def thread_local key1 = nil, key2 = nil, key3 = nil
+    obj = Thread.current[:hbase_jruby] ||= {}
+    obj = obj[key1] ||= {} if key1
+    obj = obj[key2] ||= {} if key2
+    obj = obj[key3] ||= {} if key3
+    obj
+  end
 end#Util
 end#HBase
 

data/lib/hbase-jruby/version.rb

@@ -1,5 +1,5 @@
 class HBase
 module JRuby
-  VERSION = '0.6.3'
+  VERSION = '0.6.4'
 end
 end

data/test/test_hbase.rb

@@ -26,7 +26,7 @@ class TestHBase < TestHBaseJRubyBase
     table.exists?
     assert @hbase.list.is_a?(Array)
 
-    assert_equal table.htable, Thread.current[:hbase_jruby][@hbase][TABLE]
+    assert_equal table.htable, Thread.current[:hbase_jruby][@hbase][TABLE.to_sym][:htable]
 
     assert !@hbase.closed?
     assert !table.closed?
@@ -51,7 +51,7 @@ class TestHBase < TestHBaseJRubyBase
     # Reconnect and check
     @hbase = connect
     table = @hbase[TABLE]
-    assert_equal table.htable, Thread.current[:hbase_jruby][@hbase][TABLE]
+    assert_equal table.htable, Thread.current[:hbase_jruby][@hbase][TABLE.to_sym][:htable]
 
     # FIXME: Connection is closed, we have to update @table object
     @table = @hbase.table(TABLE)
@@ -83,9 +83,9 @@ class TestHBase < TestHBaseJRubyBase
     table = hbase2[TABLE]
     assert_nil Thread.current[:hbase_jruby][hbase2]
 
-    table.htable
+    ht = table.htable
     # Thread-local htable cache has now been created
-    assert Thread.current[:hbase_jruby][hbase2][TABLE]
+    assert_equal ht, Thread.current[:hbase_jruby][hbase2][TABLE.to_sym][:htable]
 
     sleeping = {}
     mutex = Mutex.new
@@ -99,8 +99,8 @@ class TestHBase < TestHBaseJRubyBase
     sleep 0.1 while mutex.synchronize { sleeping.length } < 4
     threads.each do |t|
       assert t[:htable]
-      assert t[:hbase_jruby][hbase2][TABLE]
-      assert_equal t[:htable], t[:hbase_jruby][hbase2][TABLE]
+      assert t[:hbase_jruby][hbase2][TABLE.to_sym]
+      assert_equal t[:htable], t[:hbase_jruby][hbase2][TABLE.to_sym][:htable]
 
       t.kill
     end

data/test/test_table.rb

@@ -410,7 +410,7 @@ class TestTable < TestHBaseJRubyBase
 
       assert_equal true, @table.check(rk, a => 100).delete(c, ts, (ts - 2000).to_i, 'cf2')
       assert_equal 1, @table.versions(:all).get(rk).to_H[:c].length
-      assert_equal (ts - 1000).to_i, @table.versions(:all).get(rk).to_H[:c].keys.first / 1000
+      assert_equal((ts - 1000).to_i, @table.versions(:all).get(rk).to_H[:c].keys.first / 1000)
       assert_equal nil, @table.get(rk)[d]
 
       assert_equal true, @table.check(rk, a => 100).delete
@@ -541,5 +541,19 @@ class TestTable < TestHBaseJRubyBase
       assert e.java_exception.is_a?(java.lang.Exception)
     end
   end
+
+  def test_thread_local_cache
+    cached = @hbase[TABLE, :cache => true]
+    not_cached = @hbase[TABLE]
+
+    cached.put     next_rowkey, 'cf1:abc' => 100
+    not_cached.put next_rowkey, 'cf1:def' => 100
+
+    assert_equal ['cf1:abc'], Thread.current[:hbase_jruby][@hbase][TABLE.to_sym][:columns].keys
+    cached.close
+
+    # FIXME closing not_cached will also remove the cache
+    assert_nil Thread.current[:hbase_jruby][@hbase][TABLE.to_sym]
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hbase-jruby
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4
 platform: java
 authors:
 - Junegunn Choi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-12-05 00:00:00.000000000 Z
+date: 2014-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit