hbase-jruby 0.3.4-java → 0.3.5-java

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+0.3.5
+-----
+- Improved `Scoped#count` method
+  - KeyOnlyFilter turned out to be compatible with SingleColumnValueFilter
+  - Now takes an optional Hash: `scoped.count(cache_blocks: false, caching: 100)`
+  - Changed not to disable server-side block caching by default
+- Supports `Scoped#limit` even when `Scan.setMaxResultSize` is not implemented
+  - `Scoped#limit(nil)` will remove the previously set value
+
 0.3.4
 -----
 - Cleanup all thread-local (fiber-local) HTable references when connection is closed

data/README.md

@@ -710,11 +710,17 @@ end
 # Scoped COUNT
 #   When counting the number of rows, use `HTable::Scoped#count`
 #   instead of just iterating through the scope, as it internally
-#   minimizes amount of data fetched with KeyOnlyFilter
+#   minimizes the amount of data transfer using KeyOnlyFilter
+#   (and FirstKeyOnlyFilter when no filter is set)
 scoped.count
 
 # This should be even faster as it dramatically reduces the number of RPC calls
-scoped.caching(5000).count
+scoped.caching(1000).count
+
+# count method takes an options Hash:
+# - :caching (default: nil)
+# - :cache_blocks (default: true)
+scoped.count(caching: 5000, cache_blocks: false)
 ```
 
 ## Basic aggregation using coprocessor

data/lib/hbase-jruby/scoped.rb

@@ -16,22 +16,29 @@ class Scoped
     Scoped.send(:new, @table)
   end
 
-  # Number of rows in the scope
+  # Count the number of rows in the scope
   # @return [Fixnum, Bignum] The number of rows in the scope
-  def count
+  # @param [Hash] options Counting options
+  # @option options [Fixnum|nil] :caching The number of rows for caching that will be passed to scanners.
+  #   Use higher values for faster scan.
+  # @option options [Boolean] :cache_blocks Whether blocks should be cached for this scan
+  def count options = {}
+    options = { :caching      => nil,
+                :cache_blocks => true }.merge(options)
+
+    scan = block_given? ? filtered_scan : filtered_scan_minimum
+    scan.cache_blocks = options[:cache_blocks]
+    if options[:caching] && (@mlimit.nil? || options[:caching] < @mlimit)
+      scan.caching = options[:caching]
+    end
+
     cnt = 0
-    begin
-      if block_given?
-        scanner = htable.getScanner(filtered_scan)
-        scanner.each do |result|
-          cnt += 1 if yield(Row.send(:new, @table, result))
-        end
-      else
-        scanner = htable.getScanner(filtered_scan_minimum)
-        scanner.each { cnt += 1 }
+    if block_given?
+      iterate(scan) do |result|
+        cnt += 1 if yield(Row.send(:new, @table, result))
       end
-    ensure
-      scanner.close if scanner
+    else
+      iterate(scan) { |r| cnt += 1 }
     end
     cnt
   end
@@ -65,13 +72,8 @@ class Scoped
   def each
     return enum_for(:each) unless block_given?
 
-    begin
-      scanner = htable.getScanner(filtered_scan)
-      scanner.each do |result|
-        yield Row.send(:new, @table, result)
-      end
-    ensure
-      scanner.close if scanner
+    iterate(filtered_scan) do |result|
+      yield Row.send(:new, @table, result)
     end
   end
 
@@ -158,11 +160,13 @@ class Scoped
   end
 
   # Returns an HBase::Scoped object with the specified row number limit
-  # @param [Fixnum] rows Sets the maximum number of rows to return from scan
+  # @param [Fixnum|nil] rows Sets the maximum number of rows to return from scan
   # @return [HBase::Scoped] HBase::Scoped object with the specified row number limit
   def limit rows
-    raise ArgumentError, "Invalid limit. Must be a non-negative integer." unless rows.is_a?(Fixnum) && rows >= 0
-    spawn :@limit, rows
+    unless (rows.is_a?(Fixnum) && rows >= 0) || rows.nil?
+      raise ArgumentError, "Invalid limit. Must be a non-negative integer or nil."
+    end
+    spawn :@limit, rows, :@mlimit, nil
   end
 
   # Returns an HBase::Scoped object with the specified time range
@@ -259,6 +263,7 @@ private
     @batch    = nil
     @caching  = nil
     @limit    = nil
+    @mlimit   = nil
     @trange   = nil
     @scan_cbs = []
     @get_cbs  = []
@@ -523,11 +528,12 @@ private
 
       # Limit
       if @limit
-        # setMaxResultSize not implemented in 0.92
+        # setMaxResultSize not yet implemented in 0.94
         if scan.respond_to?(:setMaxResultSize)
           scan.setMaxResultSize(@limit)
         else
-          raise NotImplementedError, 'Scan.setMaxResultSize not implemented'
+          @mlimit = @limit
+          scan.caching = [@mlimit, @caching].compact.min
         end
       end
 
@@ -560,15 +566,23 @@ private
   # @private
   def filtered_scan_minimum
     filtered_scan.tap do |scan|
-      scan.cache_blocks = false
       scan.setMaxVersions 1
 
-      # A filter that will only return the first KV from each row
-      # A filter that will only return the key component of each KV
-      unless scan.getFilter
-        filters = [FirstKeyOnlyFilter.new, KeyOnlyFilter.new]
-        scan.setFilter FilterList.new(filters)
+      # FirstKeyOnlyFilter: A filter that will only return the first KV from each row-
+      # - Not compatible with SingleColumnValueFilter
+      # KeyOnlyFilter: A filter that will only return the key component of each KV
+      # - Compatible with SingleColumnValueFilter
+      ko = KeyOnlyFilter.new
+      if flist = scan.getFilter
+        if flist.is_a?(FilterList)
+          flist.addFilter ko
+        else
+          flist = FilterList.new([flist, ko])
+        end
+      else
+        flist = FilterList.new([ko, FirstKeyOnlyFilter.new])
       end
+      scan.setFilter flist
     end
   end
 
@@ -633,6 +647,22 @@ private
   def check_closed
     raise RuntimeError, "HBase connection is already closed" if @table.closed?
   end
+
+  def iterate scan
+    scanner = htable.getScanner(scan)
+    if @mlimit
+      scanner.each_with_index do |result, idx|
+        yield result
+        break if idx == @mlimit - 1
+      end
+    else
+      scanner.each do |result|
+        yield result
+      end
+    end
+  ensure
+    scanner.close if scanner
+  end
 end#Scoped
 end#HBase
 

data/lib/hbase-jruby/version.rb

@@ -1,5 +1,5 @@
 class HBase
 module JRuby
-  VERSION = '0.3.4'
+  VERSION = '0.3.5'
 end
 end

data/test/test_scoped.rb

@@ -81,14 +81,34 @@ class TestScoped < TestHBaseJRubyBase
     assert_equal 10,  @table.range(111..120).count
 
     # Start key ~ Stop key (inclusive) + limit
-    begin
-      assert_equal 5, @table.range(111..120).limit(5).count
-    rescue NotImplementedError
-    end
+    assert_equal 5,  @table.range(111..120).limit(5).count
+    assert_equal 5,  @table.range(111..120).limit(5).to_a.count
+    assert_equal 10, @table.range(111..120).limit(5).limit(nil).to_a.count
+    scoped = @table.range(111..120).limit(5)
+    assert_equal 5,  scoped.count
+
+    # Scan.setCaching should be called when setMaxResultSize is not implemented
+    done = false
+    scoped.with_java_scan { |scan|
+      assert scan.respond_to?(:setMaxResultSize) || scan.caching == 5
+      done = true
+    }.count
+    assert done
+
+    assert_equal 10, scoped.limit(nil).count
+
+    # Filters
+    assert_equal 1,  @table.filter('cf1:a' => 135).count
+    assert_equal 1,  @table.filter('cf2:b' => 135).count
+    assert_equal 10, @table.filter('cf1:a' => 131..140).count
+    assert_equal 10, @table.filter('cf2:b' => 131..140).count
+    assert_equal 10, @table.project(:prefix => 'a').filter('cf2:b' => 131..140).count
 
     # Start key ~ Stop key (inclusive) + filters
     assert_equal 10,  @table.range(111..150).filter('cf1:a' => 131..140).count
+    assert_equal 10,  @table.range(111..150).filter('cf2:b' => 131..140).count
     assert_equal 9,   @table.range(111..150).filter('cf1:a' => 131...140).count
+    assert_equal 9,   @table.range(111..150).filter('cf2:b' => 131...140).count
     assert_equal 2,   @table.range(111..150).filter('cf1:a' => 131...140, 'cf2:b' => 132..133).count
 
     # Count with block
@@ -522,5 +542,22 @@ class TestScoped < TestHBaseJRubyBase
       get.setTimeRange(t2.to_i * 1000, t4.to_i * 1000)
     }.get(rks).compact.count
   end
+
+  def test_count_options
+    # TODO how to confirm?
+
+    (101..150).each do |i|
+      @table.put(i, 'cf1:a' => i, 'cf2:b' => i, 'cf3:c' => i * 3)
+    end
+
+    assert_equal 50, @table.count(:cache_blocks => false)
+    assert_equal 50, @table.count(:cache_blocks => true)
+
+    assert_equal 50, @table.count(:caching => nil)
+    assert_equal 50, @table.count(:caching => 5)
+    assert_equal 50, @table.count(:caching => 500)
+    assert_equal 15, @table.limit(15).count(:caching => 500)
+    assert_equal 15, @table.limit(15).count(:caching => 500, :cache_blocks => false)
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hbase-jruby
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
   prerelease:
 platform: java
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-07-05 00:00:00.000000000 Z
+date: 2013-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit