hbase-jruby 0.2.6-java → 0.3.0-java

data/lib/hbase-jruby/schema.rb

@@ -0,0 +1,115 @@
+require 'forwardable'
+
+class HBase
+class Schema
+  extend Forwardable
+  def_delegators :@schema, :inspect, :to_s
+
+  def initialize
+    @schema = {}
+    @lookup = {}
+  end
+
+  # [cq]
+  # [cf:cq]
+  # @param [Symbol] table
+  # @param [Hash] definition
+  def []= table, definition
+    if definition.nil? || definition.empty?
+      delete table
+      return nil
+    end
+
+    unless definition.is_a?(Hash)
+      raise ArgumentError, 'invalid schema definition: Hash required'
+    end
+    definition = definition.dup.freeze
+    lookup     = empty_lookup_table
+
+    definition.each do |cf, cols|
+      unless [Symbol, String].any? { |k| cf.is_a? k }
+        raise ArgumentError,
+          "invalid schema: use String or Symbol for column family name"
+      end
+
+      # CF:CQ => Type shortcut
+      cf = cf.to_s
+      if cf.index(':')
+        cf, q = key.to_s.split ':', 2
+        cols = { q => cols }
+      else
+        raise ArgumentError, "invalid schema: expected Hash" unless cols.is_a?(Hash)
+      end
+
+      # Family => { Column => Type }
+      cols.each do |cq, type|
+        raise ArgumentError, "invalid schema" unless type.is_a?(Symbol)
+
+        # Pattern
+        case cq
+        when Regexp
+          lookup[:pattern][cq] = [cf, nil, type]
+        # Exact
+        when String, Symbol
+          cq = cq.to_s
+          cfcq = [cf, cq].join(':')
+          [cq, cq.to_sym, cfcq].each do |key|
+            lookup[:exact][key] = [cf, cq.to_sym, type]
+          end
+        else
+          raise ArgumentError, "invalid schema"
+        end
+      end
+    end
+
+    table = table.to_sym
+    @lookup[table] = lookup
+    @schema[table] = definition
+  end
+
+  # @private
+  # @param [Symbol] table
+  # @return [Array] CF, CQ, Type. When not found, nil.
+  def lookup table, col
+    return nil unless lookup = @lookup[table]
+
+    if match = lookup[:exact][col]
+      return match
+    elsif pair = lookup[:pattern].find { |k, v| col.to_s =~ k }
+      return pair[1].dup.tap { |e| e[1] = col.to_sym }
+    end
+  end
+
+  # @private
+  # @param [Symbol] table
+  def lookup_and_parse table, col
+    cf, cq, type = lookup table, col
+    cf, cq = Util.parse_column_name(cf ? [cf, cq] : col)
+    return [cf, cq, type]
+  end
+
+  # Delete schema for the table
+  # @param [Symbol] table
+  def delete table
+    table = table.to_sym
+    @lookup.delete table
+    @schema.delete table
+    nil
+  end
+
+  # @return [Hash]
+  def to_h
+    @schema
+  end
+
+private
+  def empty_lookup_table
+    {
+      :exact   => {},
+      :pattern => {},
+    }
+  end
+
+end
+end
+

data/lib/hbase-jruby/scoped/aggregation.rb

@@ -17,6 +17,20 @@ module Aggregation
       cpc = 'org.apache.hadoop.hbase.coprocessor.AggregateImplementation'
       add_coprocessor! cpc, &block unless has_coprocessor?(cpc)
     end
+
+    # Disables aggregation support for the table (asynchronous)
+    # @return [nil]
+    def disable_aggregation
+      cpc = 'org.apache.hadoop.hbase.coprocessor.AggregateImplementation'
+      remove_coprocessor cpc if has_coprocessor?(cpc)
+    end
+
+    # Disables aggregation support for the table (synchronous)
+    # @return [nil]
+    def disable_aggregation! &block
+      cpc = 'org.apache.hadoop.hbase.coprocessor.AggregateImplementation'
+      remove_coprocessor! cpc, &block if has_coprocessor?(cpc)
+    end
   end
 
   # Performs aggregation with coprocessor

data/lib/hbase-jruby/scoped.rb

@@ -24,7 +24,7 @@ class Scoped
       if block_given?
         scanner = htable.getScanner(filtered_scan)
         scanner.each do |result|
-          cnt += 1 if yield(Row.send(:new, result))
+          cnt += 1 if yield(Row.send(:new, @table, result))
         end
       else
         scanner = htable.getScanner(filtered_scan_minimum)
@@ -53,11 +53,11 @@ class Scoped
     case rowkeys
     when Array
       htable.get(rowkeys.map { |rk| getify rk }).map { |result|
-        result.isEmpty ? nil : Row.new(result)
+        result.isEmpty ? nil : Row.send(:new, @table, result)
       }
     else
       result = htable.get(getify rowkeys)
-      result.isEmpty ? nil : Row.new(result)
+      result.isEmpty ? nil : Row.send(:new, @table, result)
     end
   end
 
@@ -67,17 +67,15 @@ class Scoped
   def each
     check_closed
 
-    if block_given?
-      begin
-        scanner = htable.getScanner(filtered_scan)
-        scanner.each do |result|
-          yield Row.send(:new, result)
-        end
-      ensure
-        scanner.close if scanner
+    return enum_for(:each) unless block_given?
+
+    begin
+      scanner = htable.getScanner(filtered_scan)
+      scanner.each do |result|
+        yield Row.send(:new, @table, result)
       end
-    else
-      self
+    ensure
+      scanner.close if scanner
     end
   end
 
@@ -208,7 +206,10 @@ class Scoped
         end
       end
     end
-    spawn :@project, @project + columns
+    spawn :@project, @project + columns.map { |c|
+      cf, cq, type = @table.lookup_schema(c)
+      cf ? [cf, cq] : c
+    }
   end
 
   # Returns an HBase::Scoped object with the specified version number limit.
@@ -408,10 +409,10 @@ private
     }
   end
 
-  def filter_for cf, cq, val
+  def filter_for cf, cq, type, val
     case val
     when Range
-      min, max = [val.begin, val.end].map { |k| Util.to_bytes k }
+      min, max = [val.begin, val.end].map { |k| Util.to_typed_bytes type, k }
       FilterList.new(FilterList::Operator::MUST_PASS_ALL, [
         SingleColumnValueFilter.new(
           cf, cq,
@@ -440,7 +441,7 @@ private
               CompareFilter::CompareOp::NOT_EQUAL
             else
               if val.length == 1
-                return filter_for(cf, cq, Util.to_bytes(val))
+                return filter_for(cf, cq, nil, Util.to_typed_bytes(type, val))
               else
                 raise ArgumentError, "Unknown operator: #{op}"
               end
@@ -456,11 +457,11 @@ private
                 FilterList::Operator::MUST_PASS_ONE
               end,
               v.map { |vv|
-                SingleColumnValueFilter.new(cf, cq, operator, Util.to_bytes(vv))
+                SingleColumnValueFilter.new(cf, cq, operator, Util.to_typed_bytes(type, vv))
               }
             )
           else
-            SingleColumnValueFilter.new(cf, cq, operator, Util.to_bytes(v))
+            SingleColumnValueFilter.new(cf, cq, operator, Util.to_typed_bytes(type, v))
           end
         }
       )
@@ -474,7 +475,7 @@ private
       SingleColumnValueFilter.new(
         cf, cq,
         CompareFilter::CompareOp::EQUAL,
-        Util.to_bytes(val))
+        Util.to_typed_bytes(type, val))
     end
   end
 
@@ -593,14 +594,14 @@ private
       case f
       when Hash
         f.map { |col, val|
-          cf, cq = Util.parse_column_name col
+          cf, cq, type = @table.lookup_and_parse col
 
           case val
           when Array
             FilterList.new(FilterList::Operator::MUST_PASS_ONE,
-              val.map { |v| filter_for cf, cq, v })
+              val.map { |v| filter_for cf, cq, type, v })
           else
-            filter_for cf, cq, val
+            filter_for cf, cq, type, val
           end
         }.flatten
       when FilterBase, FilterList
@@ -622,6 +623,12 @@ private
     end
   end
 
+  # @private
+  def typed_bytes col, v
+    _, _, type = @table.lookup_schema(col)
+    Util.to_typed_bytes(type || :raw, v)
+  end
+
   def check_closed
     raise RuntimeError, "HBase connection is already closed" if @table.closed?
   end

data/lib/hbase-jruby/table.rb

@@ -44,16 +44,16 @@ class Table
    :time_range, :at
   ].each do |method|
     define_method(method) do |*args|
-      self.each.send(method, *args)
+      self.scoped.send(method, *args)
     end
   end
 
   def with_java_scan &block
-    self.each.with_java_scan(&block)
+    self.scoped.with_java_scan(&block)
   end
 
   def with_java_get &block
-    self.each.with_java_get(&block)
+    self.scoped.with_java_get(&block)
   end
 
   # Performs PUT operations
@@ -122,7 +122,9 @@ class Table
 
     htable.delete specs.map { |spec|
       rowkey, cfcq, *ts = spec
-      cf, cq = Util.parse_column_name(cfcq) if cfcq
+      if cfcq
+        cf, cq, _ = lookup_and_parse cfcq
+      end
 
       Delete.new(Util.to_bytes rowkey).tap { |del|
         if !ts.empty?
@@ -163,20 +165,30 @@ class Table
   #   @param [Hash] column_by_hash Column expression to increment amount pairs
   #   @example
   #     table.increment('a000', 'cf1:col1' => 1, 'cf1:col2' => 2)
+  # @overload increment(inc_spec)
+  #   Increase values of multiple columns from multiple rows.
+  #   @param [Hash] inc_spec { rowkey => { col => by } }
+  #   @example
+  #     table.increment 'a000' => { 'cf1:col1' => 1, 'cf1:col2' => 2 },
+  #                     'a001' => { 'cf1:col1' => 3, 'cf1:col2' => 4 }
   def increment rowkey, *args
     check_closed
 
-    if args.first.is_a?(Hash)
+    if args.empty? && rowkey.is_a?(Hash)
+      rowkey.each do |key, spec|
+        increment key, spec
+      end
+    elsif args.first.is_a?(Hash)
       cols = args.first
       htable.increment Increment.new(Util.to_bytes rowkey).tap { |inc|
         cols.each do |col, by|
-          cf, cq = Util.parse_column_name(col)
+          cf, cq, _ = lookup_and_parse col
           inc.addColumn cf, cq, by
         end
       }
     else
       col, by = args
-      cf, cq = Util.parse_column_name(col)
+      cf, cq = lookup_and_parse col
       htable.incrementColumnValue Util.to_bytes(rowkey), cf, cq, by || 1
     end
   end
@@ -184,23 +196,32 @@ class Table
   # Scan through the table
   # @yield [row] Yields each row in the scope
   # @yieldparam [HBase::Row] row
+  # @return [Enumerator]
+  def each &block
+    scoped.each(&block)
+  end
+
+  # Returns HBase::Scoped object for this table
   # @return [HBase::Scoped]
-  def each
-    check_closed
+  def scoped
+    Scoped.send(:new, self)
+  end
 
-    if block_given?
-      Scoped.send(:new, self).each { |r| yield r }
-    else
-      Scoped.send(:new, self)
-    end
+  def lookup_schema col
+    @hbase.schema.lookup @name_sym, col
+  end
+
+  def lookup_and_parse col
+    @hbase.schema.lookup_and_parse @name_sym, col
   end
 
 private
   def initialize hbase, config, htable_pool, name
-    @hbase  = hbase
-    @config = config
-    @pool   = htable_pool
-    @name   = name.to_s
+    @hbase    = hbase
+    @config   = config
+    @pool     = htable_pool
+    @name     = name.to_s
+    @name_sym = name.to_sym
   end
 
   def check_closed
@@ -210,21 +231,22 @@ private
   def putify rowkey, props
     Put.new(Util.to_bytes rowkey).tap { |put|
       props.each do |col, val|
-        cf, cq = Util.parse_column_name(col)
+        cf, cq, type = lookup_and_parse col
+
         case val
         when Hash
           val.each do |t, v|
             case t
             # Timestamp / Ruby Time
             when Time, Fixnum
-              put.add cf, cq, time_to_long(t), Util.to_bytes(v)
+              put.add cf, cq, time_to_long(t), Util.to_typed_bytes(type, v)
             # Types: :byte, :short, :int, ...
             else
-              put.add cf, cq, Util.to_bytes(t => v)
+              put.add cf, cq, Util.to_typed_bytes(t, v)
             end
           end
         else
-          put.add cf, cq, Util.to_bytes(val)
+          put.add cf, cq, Util.to_typed_bytes(type, val)
         end
       end
     }

data/lib/hbase-jruby/util.rb

@@ -65,6 +65,41 @@ module Util
       end
     end
 
+    def to_typed_bytes type, val
+      return Util.to_bytes val if type.nil?
+
+      import_java_classes!
+      case type
+      when :string, :str, :symbol, :sym
+        val.to_s.to_java_bytes
+      when :byte
+        [val].to_java(Java::byte)
+      when :boolean, :bool
+        Bytes.to_bytes val
+      when :int
+        Bytes.java_send :toBytes, [Java::int], val
+      when :short
+        Bytes.java_send :toBytes, [Java::short], val
+      when :long, :fixnum
+        Bytes.java_send :toBytes, [Java::long], val
+      when :float, :double
+        Bytes.java_send :toBytes, [Java::double], val
+      when :bigdecimal
+        case val
+        when BigDecimal
+          Bytes.java_send :toBytes, [java.math.BigDecimal], val.to_java
+        when java.math.BigDecimal
+          Bytes.java_send :toBytes, [java.math.BigDecimal], val
+        else
+          raise ArgumentError, "not BigDecimal"
+        end
+      when :raw
+        val
+      else
+        raise ArgumentError, "invalid type: #{type}"
+      end
+    end
+
     # Returns Ruby object decoded from the byte array according to the given type
     # @param [Symbol, Class] type Type to convert to
     # @param [byte[]] val Java byte array
@@ -92,7 +127,7 @@ module Util
         Bytes.to_double val
       when :boolean, :bool
         Bytes.to_boolean val
-      when :raw
+      when :raw, nil
         val
       else
         raise ArgumentError, "Invalid type: #{type}"
@@ -113,8 +148,6 @@ module Util
     # @param [Object, Array, KeyValue] col
     def parse_column_name col
       case col
-      when ColumnKey
-        return col.cf.to_java_bytes, col.cq(:raw)
       when KeyValue
         return col.getFamily, col.getQualifier
       when Array
@@ -122,8 +155,9 @@ module Util
       when '', nil
         raise ArgumentError, "Column family not specified"
       else
-        cf, cq = KeyValue.parseColumn(col.to_s.to_java_bytes)
-        cq = JAVA_BYTE_ARRAY_EMPTY if cq.nil? && col.to_s[-1, 1] == ':'
+        col = col.to_s
+        cf, cq = KeyValue.parseColumn(col.to_java_bytes)
+        cq = JAVA_BYTE_ARRAY_EMPTY if cq.nil? && col[-1, 1] == ':'
         return cf, cq
       end
     end

data/lib/hbase-jruby/version.rb

@@ -1,5 +1,5 @@
 class HBase
 module JRuby
-  VERSION = '0.2.6'
+  VERSION = '0.3.0'
 end
 end

data/lib/hbase-jruby.rb

@@ -2,19 +2,19 @@ unless RUBY_PLATFORM =~ /java/
   raise LoadError, 'Only supports JRuby'
 end
 
-require "hbase-jruby/version"
-require "hbase-jruby/dependency"
-require "hbase-jruby/util"
-require "hbase-jruby/byte_array"
-require "hbase-jruby/column_key"
-require "hbase-jruby/cell"
-require "hbase-jruby/admin"
-require "hbase-jruby/scoped/aggregation"
-require "hbase-jruby/scoped"
-require "hbase-jruby/table"
-require "hbase-jruby/table/admin"
-require "hbase-jruby/table/inspection"
-require "hbase-jruby/row"
+require 'hbase-jruby/version'
+require 'hbase-jruby/dependency'
+require 'hbase-jruby/util'
+require 'hbase-jruby/byte_array'
+require 'hbase-jruby/cell'
+require 'hbase-jruby/admin'
+require 'hbase-jruby/scoped/aggregation'
+require 'hbase-jruby/scoped'
+require 'hbase-jruby/schema'
+require 'hbase-jruby/table'
+require 'hbase-jruby/table/admin'
+require 'hbase-jruby/table/inspection'
+require 'hbase-jruby/row'
 require 'hbase-jruby/hbase'
 
 HBase.import_java_classes!

data/test/helper.rb

@@ -7,10 +7,16 @@ SimpleCov.start
 
 RECREATE = false
 
+unless defined? Enumerator
+  Enumerator = Enumerable::Enumerator
+end
+
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
 require "hbase-jruby"
-HBase.resolve_dependency!(ENV.fetch('HBASE_JRUBY_TEST_DIST'), :verbose => true)
+if dist = ENV['HBASE_JRUBY_TEST_DIST']
+  HBase.resolve_dependency!(dist, :verbose => true)
+end
 HBase.log4j = { 'log4j.threshold' => 'ERROR' }
 
 class TestHBaseJRubyBase < Test::Unit::TestCase

data/test/test_aggregation.rb

@@ -30,7 +30,7 @@ class TestAggregation < TestHBaseJRubyBase
 
     # No projection
     assert_raise(ArgumentError) { @table.aggregate(:sum) }
-    assert_raise(ArgumentError) { @table.each.aggregate(:sum) }
+    assert_raise(ArgumentError) { @table.scoped.aggregate(:sum) }
 
     # Invalid type
     assert_raise(ArgumentError) { @table.project('cf1:a').aggregate(:sum, :float) }

data/test/test_byte_array.rb

@@ -39,7 +39,7 @@ class TestByteArray < Test::Unit::TestCase
   end
 
   def test_concat
-    concat = HBase::ByteArray(100) + HBase::ByteArray(200)
+    concat = HBase::ByteArray(100) + HBase::ByteArray[200]
     assert_instance_of HBase::ByteArray, concat
     assert_equal 16, concat.to_java_bytes.to_a.length
 

data/test/test_cell.rb

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
 require 'helper'
 require 'bigdecimal'
 
-class TestCell < Test::Unit::TestCase
+class TestCell < TestHBaseJRubyBase
   import org.apache.hadoop.hbase.KeyValue
   Util = HBase::Util
 
@@ -23,9 +23,9 @@ class TestCell < Test::Unit::TestCase
       { :byte  => 100 }         => :byte
     }.each do |value, type|
       kv = KeyValue.new("rowkey".to_java_bytes, "hello".to_java_bytes, "world".to_java_bytes, ts, Util.to_bytes(value))
-      cell = HBase::Cell.new(kv)
+      cell = HBase::Cell.new(@table, kv) # FIXME
 
-      assert_equal "rowkey", cell.rowkey
+      assert_equal "rowkey", cell.rowkey(:string)
       assert_equal "hello",  cell.cf, cell.family
       assert_equal "world",  cell.cq, cell.qualifier
       assert_equal ts,       cell.ts
@@ -35,7 +35,6 @@ class TestCell < Test::Unit::TestCase
         assert_equal value, cell.send(type)
       end
       assert HBase::Util.java_bytes?(cell.raw)
-      assert_equal HBase::ColumnKey.new(:hello, :world), cell.column_key
       assert_instance_of String, cell.inspect
     end
   end
@@ -51,7 +50,7 @@ class TestCell < Test::Unit::TestCase
         KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes,  "beta".to_java_bytes,  ts, val),
         KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "beta".to_java_bytes,  ts, val),
         KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "gamma".to_java_bytes, ts, val),
-      ].map { |kv| HBase::Cell.new kv }
+      ].map { |kv| HBase::Cell.new @table, kv }
 
     assert_equal cells, cells.reverse.sort
   end