hbase-jruby 0.1.3-java → 0.1.4-java

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+0.1.4
+-----
+- Fix: Start/stop row not correctly set when byte array rowkey range specified
+- More efficient count with FirstKeyOnlyFilter
+- Added `HBase::ByteArray` method as a shorthand notation for `HBase::ByteArray.new`
+- Added `HBase::ByteArray#+` method for concatenating two byte arrays
+- Added `HBase::Util::java_bytes?` method
+- Documentation
+
 0.1.3
 -----
 - Supports Ruby 1.8 compatibility mode

data/README.md

@@ -109,7 +109,9 @@ hbase = HBase.new 'hbase.zookeeper.quorum' => 'remote-server.mydomain.net'
 
 # Extra configuration
 hbase = HBase.new 'hbase.zookeeper.quorum' => 'remote-server.mydomain.net',
-                  'hbase.client.retries.number' => 3
+                  'hbase.client.retries.number' => 3,
+                  'hbase.client.scanner.caching' => 1000,
+                  'hbase.rpc.timeout' => 120000
 
 # Close HBase connection
 hbase.close
@@ -310,7 +312,7 @@ table.truncate!
 # Atomically increase cf1:counter by one
 table.increment('rowkey1', 'cf1:counter', 1)
 
-# Atomically increase two columns by one an two respectively
+# Atomically increase two columns by one and two respectively
 table.increment('rowkey1', 'cf1:counter' => 1, 'cf1:counter2' => 2)
 ```
 
@@ -504,7 +506,7 @@ scoped.project(:prefix => 'alice').
 scoped.project(:range => 'a'...'c').
        project(:range => ['i'...'k', 'x'...'z'])
 
-# Column pagination filter (Cannot be chained. Must be called exactly once.):
+# Column pagination filter:
 #   Fetch columns within the specified intra-scan offset and limit
 scoped.project(:offset => 1000, :limit => 10)
 ```
@@ -556,6 +558,9 @@ end
 #   instead of just iterating through the scope, as it internally
 #   minimizes amount of data fetched with KeyOnlyFilter
 scoped.count
+
+# This should be even faster as it dramatically reduces the number of RPC calls
+scoped.caching(1000).count
 ```
 
 ## Basic aggregation using coprocessor
@@ -674,7 +679,7 @@ table.remove_coprocessor! cp_class_name1
 ```
 
 You can perform other types of administrative tasks
-with Native Java [HBaseAdmin object](http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html),
+with native Java [HBaseAdmin object](http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html),
 which can be obtained by `HBase#admin` method. Optionally, a block can be given
 so that the HBaseAdmin object is automatically closed at the end of the given block.
 

data/lib/hbase-jruby/byte_array.rb

@@ -1,21 +1,49 @@
 class HBase
-# @private
+class << self
+  # Shortcut method to HBase::ByteArray.new
+  # @param [Object] value
+  def ByteArray value
+    ByteArray.new value
+  end
+end
+# Boxed class for Java byte arrays
+# @!attribute [r] java
+#   @return [byte[]] Java byte array
 class ByteArray
   attr_reader :java
 
+  # @param [Object] value
   def initialize value
     @java = Util.to_bytes value
   end
 
+  # Checks if the two byte arrays are the same
+  # @param [HBase::ByteArray] other
   def eql? other
     Arrays.equals(@java, other.java)
   end
   alias == eql?
 
+  # Compares two ByteArray objects
+  # @param [HBase::ByteArray] other
   def <=> other
     Bytes.compareTo(@java, other.java)
   end
 
+  # Concats two byte arrays
+  # @param [HBase::ByteArray] other
+  def + other
+    ByteArray.new(Bytes.add @java, other.java)
+  end
+
+  # Returns the Java byte array
+  # @return [byte[]]
+  def to_java_bytes
+    @java
+  end
+
+  # Returns the first byte array whose prefix doesn't match this byte array
+  # @return [byte[]]
   def stopkey_bytes_for_prefix
     arr = @java.to_a
     csr = arr.length - 1
@@ -31,6 +59,8 @@ class ByteArray
     end
   end
 
+  # Returns a hash number for the byte array
+  # @return [Fixnum]
   def hash
     Arrays.java_send(:hashCode, [Util::JAVA_BYTE_ARRAY_CLASS], @java)
   end

data/lib/hbase-jruby/cell.rb

@@ -105,6 +105,7 @@ class Cell
     KeyValue.COMPARATOR.compare(@java, other.java)
   end
 
+  # Returns a printable version of this cell
   # @return [String]
   def inspect
     %[#{cf}:#{cq} = "#{string}"@#{ts}]

data/lib/hbase-jruby/column_key.rb

@@ -8,6 +8,8 @@ class << self
   end
 end
 # Boxed class for column keys
+# @!attribute [r] cf
+#   @return [String] The column family
 class ColumnKey
   attr_reader :cf
   alias family cf
@@ -20,12 +22,14 @@ class ColumnKey
     @cq = Util.to_bytes(cq)
   end
 
+  # Returns the column qualifer decoded as the given type
   # @param [Symbol] type
   def cq type = :string
     Util.from_bytes type, @cq
   end
   alias qualifier cq
 
+  # Checks whether if the two ColumnKeys are equal
   # @param [Object] other
   def eql? other
     other = other_as_ck(other)
@@ -33,6 +37,7 @@ class ColumnKey
   end
   alias == eql?
 
+  # Compares two ColumnKeys
   # @param [Object] other
   def <=> other
     other = other_as_ck(other)
@@ -40,10 +45,14 @@ class ColumnKey
     d != 0 ? d : Bytes.compareTo(@cq, other.cq(:raw))
   end
 
+  # Returns a hash number for this ColumnKey
+  # @return [Fixnum]
   def hash
     [@cf, Arrays.java_send(:hashCode, [Util::JAVA_BYTE_ARRAY_CLASS], @cq)].hash
   end
 
+  # Returns String representation of the column key (Qualifier decoded as a String)
+  # @return [String]
   def to_s
     [@cf, @cq.empty? ? nil : cq].compact.join(':')
   end

data/lib/hbase-jruby/result.rb

@@ -66,6 +66,7 @@ class Result
     }
   end
 
+  # Returns column values as byte arrays
   # @overload raw(column)
   #   Returns the latest column value as a byte array
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -86,6 +87,7 @@ class Result
     end
   end
 
+  # Returns all versions of column values as byte arrays in a Hash indexed by their timestamps
   # @overload raws(column)
   #   Returns all versions of column values as byte arrays in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -106,6 +108,7 @@ class Result
     end
   end
 
+  # Returns column values as Strings
   # @overload string(column)
   #   Returns the latest column value as a String
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -120,6 +123,7 @@ class Result
   end
   alias str string
 
+  # Returns all versions of column values as Strings in a Hash indexed by their timestamps
   # @overload strings(column)
   #   Returns all versions of column values as Strings in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -134,6 +138,7 @@ class Result
   end
   alias strs strings
 
+  # Returns column values as Symbols
   # @overload symbol(column)
   #   Returns the latest column value as a Symbol
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -148,6 +153,7 @@ class Result
   end
   alias sym symbol
 
+  # Returns all versions of column values as Symbols in a Hash indexed by their timestamps
   # @overload symbols(column)
   #   Returns all versions of column values as Symbols in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -162,6 +168,7 @@ class Result
   end
   alias syms symbols
 
+  # Returns column values as Fixnums
   # @overload fixnum(column)
   #   Returns the latest column value as a Fixnum
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -177,6 +184,7 @@ class Result
   alias integer fixnum
   alias int     fixnum
 
+  # Returns all versions of column values as Fixnums in a Hash indexed by their timestamps
   # @overload fixnums(column)
   #   Returns all versions of column values as Fixnums in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -192,6 +200,7 @@ class Result
   alias integers fixnums
   alias ints     fixnums
 
+  # Returns column values as Bigdecimals
   # @overload bigdecimal(column)
   #   Returns the latest column value as a BigDecimal
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -205,6 +214,7 @@ class Result
     decode_values :bigdecimal, cols
   end
 
+  # Returns all versions of column values as BigDecimals in a Hash indexed by their timestamps
   # @overload bigdecimals(column)
   #   Returns all versions of column values as BigDecimals in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -218,6 +228,7 @@ class Result
     decode_values :bigdecimal, cols, true
   end
 
+  # Returns column values as Floats
   # @overload float(column)
   #   Returns the latest column value as a Float
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -232,6 +243,7 @@ class Result
   end
   alias double float
 
+  # Returns all versions of column values as Floats in a Hash indexed by their timestamps
   # @overload floats(column)
   #   Returns all versions of column values as Floats in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -246,6 +258,7 @@ class Result
   end
   alias doubles floats
 
+  # Returns column values as Booleans
   # @overload boolean(column)
   #   Returns the latest column value as a boolean value
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -260,6 +273,7 @@ class Result
   end
   alias bool boolean
 
+  # Returns all versions of column values as Booleans in a Hash indexed by their timestamps
   # @overload booleans(column)
   #   Returns all versions of column values as boolean values in a Hash indexed by their timestamps
   #   @param [String, HBase::ColumnKey] column "FAMILY:QUALIFIER" expression or ColumnKey object.
@@ -274,6 +288,7 @@ class Result
   end
   alias bools booleans
 
+  # Compares two Result instances on their row keys
   def <=> other
     Bytes.compareTo(rowkey(:raw), other.rowkey(:raw))
   end

data/lib/hbase-jruby/scoped.rb

@@ -35,6 +35,7 @@ class Scoped
     cnt
   end
 
+  # Performs GET operations
   # @overload get(rowkey)
   #   Single GET.
   #   Gets a record with the given rowkey. If the record is not found, nil is returned.
@@ -115,7 +116,7 @@ class Scoped
   #     table.range(:prefix => ['2010', '2012'])
   def range *key_range
     if key_range.last.is_a?(Hash)
-      prefixes  = [*key_range.last[:prefix]].compact
+      prefixes  = arrayfy(key_range.last[:prefix]).compact
       raise ArgumentError,
         "Invalid range. Unknown option(s) specified." unless (key_range.last.keys - [:prefix]).empty?
       key_range = key_range[0...-1]
@@ -237,9 +238,9 @@ private
         col.each do |prop, val|
           case prop
           when :prefix
-            prefixes += [*val]
+            prefixes += arrayfy(val)
           when :range
-            ranges += val.is_a?(Array) ? val : [val]
+            ranges += arrayfy(val)
           when :limit
             limit = val
           when :offset
@@ -462,10 +463,15 @@ private
     filtered_scan.tap do |scan|
       scan.cache_blocks = false
 
+      # A filter that will only return the first KV from each row
+      # A filter that will only return the key component of each KV
+      filters = [FirstKeyOnlyFilter.new, KeyOnlyFilter.new]
       if flist = scan.getFilter
-        flist.addFilter KeyOnlyFilter.new
+        filters.each do |filter|
+          flist.addFilter filter
+        end
       else
-        scan.setFilter FilterList.new(KeyOnlyFilter.new)
+        scan.setFilter FilterList.new(filters)
       end
     end
   end
@@ -516,6 +522,17 @@ private
       end
     }.flatten
   end
+
+  def arrayfy val
+    # No range splat
+    if Util.java_bytes?(val)
+      [val]
+    elsif val.is_a?(Array)
+      val
+    else
+      [val]
+    end
+  end
 end#Scoped
 end#HBase
 

data/lib/hbase-jruby/table.rb

@@ -47,6 +47,7 @@ class Table
     !enabled?
   end
 
+  # Creates the table
   # @overload create!(column_family_name, props = {})
   #   Create the table with one column family of the given name
   #   @param [#to_s] The name of the column family
@@ -257,6 +258,7 @@ class Table
     end
   end
 
+  # Performs PUT operations
   # @overload put(rowkey, data)
   #   Put operation on a rowkey
   #   @param [Object] rowkey Rowkey
@@ -274,6 +276,7 @@ class Table
     puts.length
   end
 
+  # Deletes data
   # @overload delete(rowkey)
   #   Deletes a row with the given rowkey
   #   @param [Object] rowkey
@@ -334,6 +337,7 @@ class Table
     }
   end
 
+  # Atomically increase numeric values
   # @overload increment(rowkey, column, by)
   #   Atomically increase column value by the specified amount
   #   @param [Object] rowkey Rowkey
@@ -383,7 +387,7 @@ class Table
       (local_htables[@name] = @pool.get_table(@name))
   end
 
-  # Returns table description
+  # Returns a printable version of the table description
   # @return [String] Table description
   def inspect
     if exists?

data/lib/hbase-jruby/util.rb

@@ -6,12 +6,16 @@ module Util
   JAVA_BYTE_ARRAY_CLASS = JAVA_BYTE_ARRAY_EMPTY.java_class
 
   class << self
+    def java_bytes? v
+      v.respond_to?(:java_class) && v.java_class == JAVA_BYTE_ARRAY_CLASS
+    end
+
     # Returns byte array representation of the Ruby object
     # @param [byte[]] v
     # @return [byte[]]
     def to_bytes v
       case v
-      when String
+      when String, ByteArray
         v.to_java_bytes
       when Fixnum
         Bytes.java_send :toBytes, [Java::long], v
@@ -30,7 +34,7 @@ module Util
       when java.math.BigDecimal
         Bytes.java_send :toBytes, [java.math.BigDecimal], v
       else
-        if v.respond_to?(:java_class) && v.java_class == JAVA_BYTE_ARRAY_CLASS
+        if java_bytes?(v)
           v
         else
           raise ArgumentError.new("Don't know how to convert #{v.class} into Java bytes")
@@ -94,6 +98,7 @@ module Util
       end
     end
 
+    # Import Java classes (Prerequisite for classes in hbase-jruby)
     # @return [nil]
     def import_java_classes!
       imp = lambda { |base, classes|
@@ -161,6 +166,7 @@ module Util
           org.apache.hadoop.hbase.filter.FilterBase
           org.apache.hadoop.hbase.filter.FilterList
           org.apache.hadoop.hbase.filter.KeyOnlyFilter
+          org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter
           org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter
           org.apache.hadoop.hbase.filter.PrefixFilter
           org.apache.hadoop.hbase.filter.RegexStringComparator

data/lib/hbase-jruby/version.rb

@@ -1,5 +1,5 @@
 class HBase
 module JRuby
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end
 end

data/test/test_byte_array.rb

@@ -37,4 +37,13 @@ class TestByteArray < Test::Unit::TestCase
     assert_equal 1, hash[ HBase::ByteArray.new("Hello") ]
     assert_equal 2, hash[ HBase::ByteArray.new("World".to_java_bytes) ]
   end
+
+  def test_concat
+    concat = HBase::ByteArray(100) + HBase::ByteArray(200)
+    assert_instance_of HBase::ByteArray, concat
+    assert_equal 16, concat.to_java_bytes.to_a.length
+
+    assert_equal 100, HBase::Util.from_bytes( :fixnum, concat.to_java_bytes.to_a[0, 8].to_java(Java::byte) )
+    assert_equal 200, HBase::Util.from_bytes( :fixnum, concat.java.to_a[8, 8].to_java(Java::byte) )
+  end
 end

data/test/test_scoped.rb

@@ -358,5 +358,25 @@ class TestScoped < TestHBaseJRubyBase
     assert_equal 28, @table.filter('cf1:a' => ['aa', 'cc', /^g/]).count
     assert_equal 54, @table.filter('cf1:a' => ['aa', 'cc', /^g/, { :gte => 'xa', :lt => 'y'}]).count
   end
+
+  def test_java_bytes_prefix
+    (1..100).each do |i|
+      (1..100).each do |j|
+        @table.put((HBase::ByteArray(i) + HBase::ByteArray(j)).to_java_bytes, 'cf1:a' => i * j)
+      end
+    end
+
+    assert_equal 100, @table.range(:prefix => HBase::ByteArray(50)).count
+    assert_equal 100, @table.range(:prefix => HBase::ByteArray(50).to_java_bytes).count
+    assert_equal 200, @table.range(HBase::ByteArray(50), HBase::ByteArray(52)).count
+    assert_equal 1,   @table.range(:prefix => (HBase::ByteArray(50) + HBase::ByteArray(50))).count
+
+    assert_equal 2,   @table.range(:prefix => [
+                                   (HBase::ByteArray(50) + HBase::ByteArray(50)).java,
+                                   (HBase::ByteArray(50) + HBase::ByteArray(51)).java ]).count
+
+    # Fails on 0.1.3
+    assert_equal 1,   @table.range(:prefix => (HBase::ByteArray(50) + HBase::ByteArray(50)).java).count
+  end
 end
 

data/test/test_util.rb

@@ -14,12 +14,14 @@ class TestUtil < Test::Unit::TestCase
 
     [:fixnum, :int, :integer].each do |type|
       assert_equal 100, Util.from_bytes( type, Util.to_bytes(100) )
+      assert_equal 100, Util.from_bytes( type, Util.to_bytes(HBase::ByteArray(100)) )
     end
     [:float, :double].each do |type|
       assert_equal 3.14, Util.from_bytes( type, Util.to_bytes(3.14) )
     end
     [:string, :str].each do |type|
       assert_equal "Hello", Util.from_bytes( type, Util.to_bytes("Hello") )
+      assert_equal "Hello", Util.from_bytes( type, Util.to_bytes(HBase::ByteArray("Hello")) )
     end
     [:bool, :boolean].each do |type|
       assert_equal true,  Util.from_bytes( type, Util.to_bytes(true) )
@@ -46,8 +48,8 @@ class TestUtil < Test::Unit::TestCase
   end
 
   def test_parse_column_name
-    assert_equal ['abc', 'def'],  parse_to_str('abc:def') 
-    assert_equal ['abc', 'def:'], parse_to_str('abc:def:') 
+    assert_equal ['abc', 'def'],  parse_to_str('abc:def')
+    assert_equal ['abc', 'def:'], parse_to_str('abc:def:')
     assert_equal ['abc', ''],     parse_to_str('abc:')
     assert_equal ['abc', nil],    parse_to_str('abc')
     assert_equal ['abc', ':::'],  parse_to_str('abc::::')
@@ -72,6 +74,16 @@ class TestUtil < Test::Unit::TestCase
     assert_equal [97, 97, 97, 0], Util.append_0("aaa".to_java_bytes).to_a
   end
 
+  def test_java_bytes
+    ["Hello", 1234, :symbol].each do |v|
+      assert_false Util.java_bytes?(v)
+    end
+
+    ["Hello".to_java_bytes, Util.to_bytes(1234), Util.to_bytes(:symbol)].each do |v|
+      assert Util.java_bytes?(v)
+    end
+  end
+
 private
   def parse_to_str v, type = :string
     Util.parse_column_name(v).map { |e| Util.from_bytes type, e }

metadata

@@ -2,7 +2,7 @@
 name: hbase-jruby
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.1.3
+  version: 0.1.4
 platform: java
 authors:
 - Junegunn Choi