hbase-jruby 0.1.6-java → 0.2.0-java

data/lib/hbase-jruby/table/inspection.rb

@@ -0,0 +1,109 @@
+class HBase
+class Table
+  # Returns a read-only org.apache.hadoop.hbase.HTableDescriptor object
+  # @return [org.apache.hadoop.hbase.client.UnmodifyableHTableDescriptor]
+  def descriptor
+    htable.get_table_descriptor
+  end
+
+  # Returns table properties
+  # @return [Hash]
+  def properties
+    desc = descriptor
+    {}.tap { |props|
+      TABLE_PROPERTIES.each do |prop, gs|
+        get = gs[:get]
+        if get && desc.respond_to?(get)
+          props[prop] = parse_property desc.send get
+        end
+      end
+    }
+  end
+
+  # Returns raw String-to-String map of table properties
+  # @return [Hash]
+  def raw_properties
+    parse_raw_map descriptor.values
+  end
+
+  # Returns properties of column families indexed by family name
+  # @return [Hash]
+  def families
+    {}.tap { |ret|
+      descriptor.families.each do |family|
+        name = family.name_as_string
+        ret[name] =
+          {}.tap { |props|
+            COLUMN_PROPERTIES.each do |prop, gs|
+              get = gs[:get]
+              if get && family.respond_to?(get)
+                props[prop] = parse_property family.send get
+              end
+            end
+          }
+      end
+    }
+  end
+
+  # Returns raw String-to-String map of column family properties indexed by name
+  # @return [Hash]
+  def raw_families
+    {}.tap { |ret|
+      descriptor.families.each do |family|
+        name = family.name_as_string
+        ret[name] = parse_raw_map family.values
+      end
+    }
+  end
+
+  # Returns region information
+  # @return [Hash]
+  def regions
+    with_admin do |admin|
+      _regions admin
+    end
+  end
+
+  # Returns a printable version of the table description
+  # @return [String] Table description
+  def inspect
+    if exists?
+      descriptor.toStringCustomizedValues
+    else
+      # FIXME
+      "{NAME => '#{@name}'}"
+    end
+  end
+
+private
+  def _regions admin
+    admin.getTableRegions(@name.to_java_bytes).map { |ri|
+      {}.tap { |r|
+        r[:name]      = ri.region_name
+        r[:id]        = ri.region_id
+        r[:start_key] = nil_if_empty ri.start_key
+        r[:end_key]   = nil_if_empty ri.end_key
+        r[:root]      = ri.is_root_region
+        r[:meta]      = ri.is_meta_region
+        r[:online]    = !ri.is_offline
+      }
+    }
+  end
+
+  def nil_if_empty v
+    v.empty? ? nil : v
+  end
+
+  def parse_property v
+    if v.is_a?(java.lang.Enum)
+      v.to_s
+    else
+      v
+    end
+  end
+
+  def parse_raw_map m
+    Hash[ m.keys.map { |e| e.get.to_s }.zip m.values.map { |e| e.get.to_s } ]
+  end
+end#Table
+end#HBase

data/lib/hbase-jruby/table.rb

@@ -13,234 +13,27 @@ class Table
   include Enumerable
   include Admin
   include Scoped::Aggregation::Admin
+  include HBase::Util
 
-  # Returns a read-only org.apache.hadoop.hbase.HTableDescriptor object
-  # @return [org.apache.hadoop.hbase.client.UnmodifyableHTableDescriptor]
-  def descriptor
-    htable.get_table_descriptor
+  # (INTERNAL) Returns the underlying org.apache.hadoop.hbase.client.HTable object (local to current thread)
+  # @return [org.apache.hadoop.hbase.client.HTable]
+  def htable
+    check_closed
+
+    local_htables = Thread.current[:htable] ||= {}
+    local_htables[@name] ||= @pool.get_table(@name)
   end
 
-  # Closes the table and returns HTable object back to the HTablePool.
+  # @deprecated
   # @return [nil]
   def close
-    Thread.current[:htable] ||= {}
-    ht = Thread.current[:htable].delete(@name)
-    ht.close if ht
     nil
   end
 
-  # Checks if the table of the name exists
-  # @return [true, false] Whether table exists
-  def exists?
-    with_admin { |admin| admin.tableExists @name }
-  end
-
-  # Checks if the table is enabled
-  # @return [true, false] Whether table is enabled
-  def enabled?
-    with_admin { |admin| admin.isTableEnabled(@name) }
-  end
-
-  # Checks if the table is disabled
-  # @return [true, false] Whether table is disabled
-  def disabled?
-    !enabled?
-  end
-
-  # Creates the table
-  # @overload create!(column_family_name, props = {})
-  #   Create the table with one column family of the given name
-  #   @param [#to_s] The name of the column family
-  #   @param [Hash] props Table properties
-  #   @return [nil]
-  # @overload create!(column_family_hash, props = {})
-  #   Create the table with the specified column families
-  #   @param [Hash] Column family properties
-  #   @param [Hash] props Table properties
-  #   @return [nil]
-  #   @example
-  #     table.create!(
-  #       # Column family with default options
-  #       :cf1 => {},
-  #       # Another column family with custom properties
-  #       :cf2 => {
-  #         :blockcache         => true,
-  #         :blocksize          => 128 * 1024,
-  #         :bloomfilter        => :row,
-  #         :compression        => :snappy,
-  #         :in_memory          => true,
-  #         :keep_deleted_cells => true,
-  #         :min_versions       => 2,
-  #         :replication_scope  => 0,
-  #         :ttl                => 100,
-  #         :versions           => 5
-  #       }
-  #     )
-  # @overload create!(table_descriptor)
-  #   Create the table with the given HTableDescriptor
-  #   @param [org.apache.hadoop.hbase.HTableDescriptor] Table descriptor
-  #   @return [nil]
-  def create! desc, props = {}
-    todo = nil
-    with_admin do |admin|
-      raise RuntimeError, 'Table already exists' if admin.tableExists(@name)
-
-      case desc
-      when HTableDescriptor
-        patch_table_descriptor! desc, props
-        admin.createTable desc
-      when Symbol, String
-        todo = lambda { create!({desc => {}}, props) }
-      when Hash
-        htd = HTableDescriptor.new(@name.to_java_bytes)
-        patch_table_descriptor! htd, props
-        desc.each do |name, opts|
-          htd.addFamily hcd(name, opts)
-        end
-
-        admin.createTable htd
-      else
-        raise ArgumentError, 'Invalid table description'
-      end
-    end
-    todo.call if todo # Avoids mutex relocking
-  end
-
-  # Alters the table (synchronous)
-  # @param [Hash] props Table properties
-  # @return [nil]
-  # @example
-  #   table.alter!(
-  #     :max_filesize       => 512 * 1024 ** 2,
-  #     :memstore_flushsize =>  64 * 1024 ** 2,
-  #     :readonly           => false,
-  #     :deferred_log_flush => true
-  #   )
-  def alter! props, &block
-    _alter props, true, &block
-  end
-
-  # Alters the table (asynchronous)
-  # @see HBase::Table#alter!
-  def alter props
-    _alter props, false
-  end
-
-  # Adds the column family (synchronous)
-  # @param [#to_s] name The name of the column family
-  # @param [Hash] opts Column family properties
-  # @return [nil]
-  def add_family! name, opts, &block
-    _add_family name, opts, true, &block
-  end
-
-  # Adds the column family (asynchronous)
-  # @see HBase::Table#add_family!
-  def add_family name, opts
-    _add_family name, opts, false
-  end
-
-  # Alters the column family
-  # @param [#to_s] name The name of the column family
-  # @param [Hash] opts Column family properties
-  # @return [nil]
-  def alter_family! name, opts, &block
-    _alter_family name, opts, true, &block
-  end
-
-  # Alters the column family (asynchronous)
-  # @see HBase::Table#alter_family!
-  def alter_family name, opts
-    _alter_family name, opts, false
-  end
-
-  # Removes the column family
-  # @param [#to_s] name The name of the column family
-  # @return [nil]
-  def delete_family! name, &block
-    _delete_family name, true, &block
-  end
-
-  # Removes the column family (asynchronous)
-  # @see HBase::Table#delete_family!
-  def delete_family name
-    _delete_family name, false
-  end
-
-  # Adds the table coprocessor to the table
-  # @param [String] class_name Full class name of the coprocessor
-  # @param [Hash] props Coprocessor properties
-  # @option props [String] path The path of the JAR file
-  # @option props [Fixnum] priority Coprocessor priority
-  # @option props [Hash<#to_s, #to_s>] params Arbitrary key-value parameter pairs passed into the coprocessor
-  def add_coprocessor! class_name, props = {}, &block
-    _add_coprocessor class_name, props, true, &block
-  end
-
-  # Adds the table coprocessor to the table (asynchronous)
-  def add_coprocessor class_name, props = {}
-    _add_coprocessor class_name, props, false
-  end
-
-  # Removes the coprocessor from the table.
-  # @param [String] class_name Full class name of the coprocessor
-  # @return [nil]
-  def remove_coprocessor! class_name, &block
-    _remove_coprocessor class_name, true, &block
-  end
-
-  # Removes the coprocessor from the table (asynchronous)
-  # @see HBase::Table#remove_coprocessor!
-  def remove_coprocessor class_name
-    _remove_coprocessor class_name, false
-  end
-
-  # Return if the table has the coprocessor of the given class name
-  # @param [String] class_name Full class name of the coprocessor
-  # @return [true, false]
-  def has_coprocessor? class_name
-    descriptor.hasCoprocessor(class_name)
-  end
-
-  # Enables the table
-  # @return [nil]
-  def enable!
-    with_admin do |admin|
-      admin.enableTable @name unless admin.isTableEnabled(@name)
-    end
-  end
-
-  # Disables the table
-  # @return [nil]
-  def disable!
-    with_admin do |admin|
-      admin.disableTable @name if admin.isTableEnabled(@name)
-    end
-  end
-
-  # Truncates the table by dropping it and recreating it.
-  # @return [nil]
-  def truncate!
-    htd = htable.get_table_descriptor
-    drop!
-    create! htd
-  end
-
-  # Drops the table
-  # @return [nil]
-  def drop!
-    with_admin do |admin|
-      raise RuntimeError, 'Table does not exist' unless admin.tableExists @name
-
-      admin.disableTable @name if admin.isTableEnabled(@name)
-      admin.deleteTable  @name
-      close
-    end
-  end
-
   [:get, :count, :aggregate,
    :range, :project, :filter, :while,
-   :limit, :versions, :caching, :batch
+   :limit, :versions, :caching, :batch,
+   :time_range, :at
   ].each do |method|
     define_method(method) do |*args|
       self.each.send(method, *args)
@@ -286,11 +79,11 @@ class Table
   #   @return [nil]
   #   @example
   #     table.delete('a000', 'cf1:col1')
-  # @overload delete(rowkey, column, timestamp)
-  #   Deletes a version of a column
+  # @overload delete(rowkey, column, *timestamps)
+  #   Deletes specified versions of a column
   #   @param [Object] rowkey
   #   @param [String, Array] column Column expression in String "FAMILY:QUALIFIER", or in Array [FAMILY, QUALIFIER]
-  #   @param [Fixnum] timestamp Timestamp.
+  #   @param [*Fixnum] timestamps Timestamps.
   #   @return [nil]
   #   @example
   #     table.delete('a000', 'cf1:col1', 1352978648642)
@@ -314,7 +107,7 @@ class Table
       Delete.new(Util.to_bytes rowkey).tap { |del|
         if !ts.empty?
           ts.each do |t|
-            del.deleteColumn cf, cq, t
+            del.deleteColumn cf, cq, time_to_long(t)
           end
         elsif cq
           # Delete all versions
@@ -358,9 +151,12 @@ class Table
   end
 
   # Scan through the table
-  # @yield [HBase::Result] Yields each row in the scope
+  # @yield [row] Yields each row in the scope
+  # @yieldparam [HBase::Result] row
   # @return [HBase::Scoped]
   def each
+    check_closed
+
     if block_given?
       Scoped.send(:new, self).each { |r| yield r }
     else
@@ -368,40 +164,16 @@ class Table
     end
   end
 
-  # Returns the underlying org.apache.hadoop.hbase.client.HTable object (local to current thread)
-  # @return [org.apache.hadoop.hbase.client.HTable]
-  def htable
-    # @htable ||= @pool.get_table(@name)
-    (local_htables = Thread.current[:htable] ||= {})[@name] ||
-      (local_htables[@name] = @pool.get_table(@name))
-  end
-
-  # Returns a printable version of the table description
-  # @return [String] Table description
-  def inspect
-    if exists?
-      htable.get_table_descriptor.to_s
-    else
-      # FIXME
-      "{NAME => '#{@name}'}"
-    end
-  end
-
 private
-  def initialize config, htable_pool, name
-    @config  = config
-    @pool    = htable_pool
-    @name    = name.to_s
-    @htable  = nil
+  def initialize hbase, config, htable_pool, name
+    @hbase  = hbase
+    @config = config
+    @pool   = htable_pool
+    @name   = name.to_s
   end
 
-  def while_disabled admin
-    begin
-      admin.disableTable @name if admin.isTableEnabled(@name)
-      yield
-    ensure
-      admin.enableTable @name
-    end
+  def check_closed
+    raise RuntimeError, "HBase connection is already closed" if @hbase.closed?
   end
 
   def putify rowkey, props
@@ -412,9 +184,9 @@ private
         when Hash
           val.each do |t, v|
             case t
-            # Timestamp
-            when Fixnum
-              put.add cf, cq, t, Util.to_bytes(v)
+            # Timestamp / Ruby Time
+            when Time, Fixnum
+              put.add cf, cq, time_to_long(t), Util.to_bytes(v)
             # Types: :byte, :short, :int, ...
             else
               put.add cf, cq, Util.to_bytes(t => v)
@@ -426,137 +198,6 @@ private
       end
     }
   end
-
-  def hcd name, opts
-    method_map = {
-      :blockcache          => :setBlockCacheEnabled,
-      :blocksize           => :setBlocksize,
-      :bloomfilter         => :setBloomFilterType,
-      :compression         => :setCompressionType,
-      :data_block_encoding => :setDataBlockEncoding,
-      :encode_on_disk      => :setEncodeOnDisk,
-      :in_memory           => :setInMemory,
-      :keep_deleted_cells  => :setKeepDeletedCells,
-      :min_versions        => :setMinVersions,
-      :replication_scope   => :setScope,
-      :ttl                 => :setTimeToLive,
-      :versions            => :setMaxVersions,
-    }
-    HColumnDescriptor.new(name.to_s).tap do |hcd|
-      opts.each do |key, val|
-        if method_map[key]
-          hcd.send method_map[key],
-            ({
-              :bloomfilter => proc { |v|
-                const_shortcut StoreFile::BloomType, v, "Invalid bloom filter type"
-              },
-              :compression => proc { |v|
-                const_shortcut Compression::Algorithm, v, "Invalid compression algorithm"
-              }
-            }[key] || proc { |a| a }).call(val)
-        else
-          raise ArgumentError, "Invalid option: #{key}"
-        end
-      end#opts
-    end
-  end
-
-  def const_shortcut base, v, message
-    vs = v.to_s.upcase
-    # const_get doesn't work with symbols in 1.8 compatibility mode
-    if base.constants.map { |c| base.const_get c }.any? { |cv| v == cv }
-      v
-    elsif base.constants.map(&:to_s).include?(vs)
-      base.const_get vs
-    else
-      raise ArgumentError, [message, v.to_s].join(': ')
-    end
-  end
-
-  def patch_table_descriptor! htd, props
-    props.each do |key, value|
-      method = {
-        :max_filesize       => :setMaxFileSize,
-        :readonly           => :setReadOnly,
-        :memstore_flushsize => :setMemStoreFlushSize,
-        :deferred_log_flush => :setDeferredLogFlush
-      }[key]
-      raise ArgumentError, "Invalid table property: #{key}" unless method
-
-      htd.send method, value
-    end
-    htd
-  end
-
-  def _alter props, bang, &block
-    with_admin do |admin|
-      htd = admin.get_table_descriptor(@name.to_java_bytes)
-      patch_table_descriptor! htd, props
-      while_disabled(admin) do
-        admin.modifyTable @name.to_java_bytes, htd
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
-
-  def _add_family name, opts, bang, &block
-    with_admin do |admin|
-      while_disabled(admin) do
-        admin.addColumn @name, hcd(name.to_s, opts)
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
-
-  def _alter_family name, opts, bang, &block
-    with_admin do |admin|
-      while_disabled(admin) do
-        admin.modifyColumn @name, hcd(name.to_s, opts)
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
-
-  def _delete_family name, bang, &block
-    with_admin do |admin|
-      while_disabled(admin) do
-        admin.deleteColumn @name, name.to_s
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
-
-  def _add_coprocessor class_name, props = {}, bang, &block
-    with_admin do |admin|
-      while_disabled(admin) do
-
-        htd = admin.get_table_descriptor(@name.to_java_bytes)
-        if props.empty?
-          htd.addCoprocessor class_name
-        else
-          path, priority, params = props.values_at :path, :priority, :params
-          params = Hash[ params.map { |k, v| [k.to_s, v.to_s] } ]
-          htd.addCoprocessor class_name, path, priority || Coprocessor::PRIORITY_USER, params
-        end
-        admin.modifyTable @name.to_java_bytes, htd
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
-
-  def _remove_coprocessor name, bang, &block
-    unless org.apache.hadoop.hbase.HTableDescriptor.respond_to?(:removeCoprocessor)
-      raise NotImplementedError, "org.apache.hadoop.hbase.HTableDescriptor.removeCoprocessor not implemented"
-    end
-    with_admin do |admin|
-      while_disabled(admin) do
-        htd = admin.get_table_descriptor(@name.to_java_bytes)
-        htd.removeCoprocessor name
-        admin.modifyTable @name.to_java_bytes, htd
-        wait_async_admin(admin, &block) if bang
-      end
-    end
-  end
 end#Table
 end#HBase
 

data/lib/hbase-jruby/util.rb

@@ -15,6 +15,8 @@ module Util
     # @return [byte[]]
     def to_bytes v
       case v
+      when Array
+        v.to_java(Java::byte)
       when String, ByteArray
         v.to_java_bytes
       when Fixnum
@@ -47,7 +49,7 @@ module Util
           Bytes.java_send :toBytes, [Java::int], val
         when :short
           Bytes.java_send :toBytes, [Java::short], val
-        when :long, :fixnum 
+        when :long, :fixnum
           Bytes.java_send :toBytes, [Java::long], val
         else
           raise ArgumentError, "Invalid value format"
@@ -123,6 +125,19 @@ module Util
       end
     end
   end
+
+private
+  # @private
+  def time_to_long ts
+    case ts
+    when Fixnum
+      ts
+    when Time
+      (ts.to_f * 1000).to_i
+    else
+      raise ArgumentError, "Invalid time format"
+    end
+  end
 end#Util
 end#HBase
 

data/lib/hbase-jruby/version.rb

@@ -1,5 +1,5 @@
 class HBase
 module JRuby
-  VERSION = "0.1.6"
+  VERSION = "0.2.0"
 end
 end

data/lib/hbase-jruby.rb

@@ -12,6 +12,8 @@ require "hbase-jruby/admin"
 require "hbase-jruby/scoped/aggregation"
 require "hbase-jruby/scoped"
 require "hbase-jruby/table"
+require "hbase-jruby/table/admin"
+require "hbase-jruby/table/inspection"
 require "hbase-jruby/result"
 require 'hbase-jruby/hbase'
 

data/test/helper.rb

@@ -8,13 +8,10 @@ SimpleCov.start
 RECREATE = false
 
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-require "hbase-jruby"
 
-# Required
-unless HBase.resolve_dependency!(ENV.fetch 'HBASE_JRUBY_TEST_DIST').all? { |f| File.exists? f }
-  puts "Invalid return value from HBase.resolve_dependency!"
-  exit 1
-end
+require "hbase-jruby"
+HBase.resolve_dependency!(ENV.fetch('HBASE_JRUBY_TEST_DIST'), :verbose => true)
+HBase.log4j = { 'log4j.threshold' => 'ERROR' }
 
 class TestHBaseJRubyBase < Test::Unit::TestCase
   TABLE = 'test_hbase_jruby'
@@ -36,7 +33,7 @@ class TestHBaseJRubyBase < Test::Unit::TestCase
     @table.create!(
       :cf1 => { :compression => :none, :bloomfilter => :row },
       :cf2 => { :bloomfilter => :rowcol },
-      :cf3 => { :versions    => 1, :bloomfilter => org.apache.hadoop.hbase.regionserver.StoreFile::BloomType::ROWCOL }
+      :cf3 => { :versions => 1, :bloomfilter => :rowcol }
     ) unless @table.exists?
     @table.enable! if @table.disabled?
 

data/test/test_cell.rb

@@ -31,12 +31,12 @@ class TestCell < Test::Unit::TestCase
       assert_instance_of String, cell.inspect
     end
   end
-  
+
   def test_order
     ts = Time.now.to_i * 1000
 
     val = "val".to_java_bytes
-    cells = 
+    cells =
       [
         KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes,  "alpha".to_java_bytes, ts, val),
         KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes,  "alpha".to_java_bytes, ts - 1000, val),