hbase-jruby 0.1.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,171 @@
1
+ require 'bigdecimal'
2
+
3
+ class HBase
4
+ module Util
5
+ JAVA_BYTE_ARRAY_EMPTY = [].to_java(Java::byte)
6
+ JAVA_BYTE_ARRAY_CLASS = JAVA_BYTE_ARRAY_EMPTY.java_class
7
+
8
+ class << self
9
+ # Returns byte array representation of the Ruby object
10
+ # @param [byte[]] v
11
+ # @return [byte[]]
12
+ def to_bytes v
13
+ case v
14
+ when String
15
+ v.to_java_bytes
16
+ when Fixnum
17
+ Bytes.java_send :toBytes, [Java::long], v
18
+ when Symbol
19
+ v.to_s.to_java_bytes
20
+ when Float
21
+ Bytes.java_send :toBytes, [Java::double], v
22
+ when true, false, ByteBuffer
23
+ Bytes.to_bytes v
24
+ when nil
25
+ ''.to_java_bytes
26
+ when Bignum
27
+ Bytes.java_send :toBytes, [java.math.BigDecimal], java.math.BigDecimal.new(v.to_s)
28
+ when BigDecimal
29
+ Bytes.java_send :toBytes, [java.math.BigDecimal], v.to_java
30
+ when java.math.BigDecimal
31
+ Bytes.java_send :toBytes, [java.math.BigDecimal], v
32
+ else
33
+ if v.respond_to?(:java_class) && v.java_class == JAVA_BYTE_ARRAY_CLASS
34
+ v
35
+ else
36
+ raise ArgumentError.new("Don't know how to convert #{v.class} into Java bytes")
37
+ end
38
+ end
39
+ end
40
+
41
+ # Returns Ruby object decoded from the byte array according to the given type
42
+ # @param [Symbol, Class] type Type to convert to
43
+ # @param [byte[]] val Java byte array
44
+ # @return [Object]
45
+ def from_bytes type, val
46
+ return nil if val.nil?
47
+
48
+ case type
49
+ when :string, :str
50
+ Bytes.to_string val
51
+ when :fixnum, :int, :integer
52
+ Bytes.to_long val
53
+ when :symbol, :sym
54
+ Bytes.to_string(val).to_sym
55
+ when :bignum, :bigint, :biginteger
56
+ BigDecimal.new(Bytes.to_big_decimal(val).to_s).to_i
57
+ when :bigdecimal
58
+ BigDecimal.new(Bytes.to_big_decimal(val).to_s)
59
+ when :float, :double
60
+ Bytes.to_double val
61
+ when :boolean, :bool
62
+ Bytes.to_boolean val
63
+ when :raw
64
+ val
65
+ else
66
+ raise ArgumentError, "Invalid type: #{type}"
67
+ end
68
+ end
69
+
70
+ # Returns a byte array with a trailing '0' byte
71
+ # @param [byte[]] v
72
+ # @return [byte[]]
73
+ def append_0 v
74
+ baos = java.io.ByteArrayOutputStream.new
75
+ baos.write v, 0, v.length
76
+ baos.write 0
77
+ baos.toByteArray
78
+ end
79
+
80
+ # Extracts a byte array pair of column family and column qualifier from the given object
81
+ # @param [Object, Array, KeyValue] col
82
+ def parse_column_name col
83
+ case col
84
+ when ColumnKey
85
+ return col.cf.to_java_bytes, col.cq(:raw)
86
+ when KeyValue
87
+ return col.getFamily, col.getQualifier
88
+ when Array
89
+ return to_bytes(col[0]), to_bytes(col[1])
90
+ when '', nil
91
+ raise ArgumentError, "Column family not specified"
92
+ else
93
+ cf, cq = KeyValue.parseColumn(col.to_s.to_java_bytes)
94
+ cq = JAVA_BYTE_ARRAY_EMPTY if cq.nil? && col.to_s[-1] == ':'
95
+ return cf, cq
96
+ end
97
+ end
98
+
99
+ # @return [nil]
100
+ def import_java_classes!
101
+ imp = lambda { |base, classes|
102
+ base.class_eval do
103
+ classes.each do |klass|
104
+ begin
105
+ import klass
106
+ rescue NameError => e
107
+ warn e
108
+ end
109
+ end
110
+ end
111
+ }
112
+
113
+ @@imported ||= begin
114
+ imp.call HBase, %w[
115
+ org.apache.hadoop.hbase.HBaseConfiguration
116
+ org.apache.hadoop.hbase.client.HBaseAdmin
117
+ org.apache.hadoop.hbase.client.HTablePool
118
+ org.apache.hadoop.hbase.client.HConnectionManager
119
+ ]
120
+
121
+ imp.call HBase::Util, %w[
122
+ java.nio.ByteBuffer
123
+ org.apache.hadoop.hbase.util.Bytes
124
+ org.apache.hadoop.hbase.KeyValue
125
+ ]
126
+
127
+ imp.call HBase::ByteArray, %w[
128
+ java.util.Arrays
129
+ org.apache.hadoop.hbase.util.Bytes
130
+ ]
131
+
132
+ imp.call HBase::Cell, %w[
133
+ org.apache.hadoop.hbase.KeyValue
134
+ ]
135
+
136
+ imp.call HBase::ColumnKey, %w[
137
+ java.util.Arrays
138
+ org.apache.hadoop.hbase.util.Bytes
139
+ ]
140
+
141
+ imp.call HBase::Table, %w[
142
+ org.apache.hadoop.hbase.HColumnDescriptor
143
+ org.apache.hadoop.hbase.HTableDescriptor
144
+ org.apache.hadoop.hbase.client.Delete
145
+ org.apache.hadoop.hbase.client.Increment
146
+ org.apache.hadoop.hbase.client.Put
147
+ org.apache.hadoop.hbase.io.hfile.Compression
148
+ org.apache.hadoop.hbase.regionserver.StoreFile
149
+ ]
150
+
151
+ imp.call HBase::Scoped, %w[
152
+ org.apache.hadoop.hbase.client.Get
153
+ org.apache.hadoop.hbase.client.Scan
154
+ org.apache.hadoop.hbase.filter.BinaryComparator
155
+ org.apache.hadoop.hbase.filter.ColumnPaginationFilter
156
+ org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter
157
+ org.apache.hadoop.hbase.filter.ColumnRangeFilter
158
+ org.apache.hadoop.hbase.filter.CompareFilter
159
+ org.apache.hadoop.hbase.filter.FilterBase
160
+ org.apache.hadoop.hbase.filter.FilterList
161
+ org.apache.hadoop.hbase.filter.KeyOnlyFilter
162
+ org.apache.hadoop.hbase.filter.PrefixFilter
163
+ org.apache.hadoop.hbase.filter.RowFilter
164
+ org.apache.hadoop.hbase.filter.SingleColumnValueFilter
165
+ ]
166
+ end
167
+ end
168
+ end
169
+ end#Util
170
+ end#HBase
171
+
@@ -0,0 +1,5 @@
1
+ class HBase
2
+ module JRuby
3
+ VERSION = "0.1.1"
4
+ end
5
+ end
@@ -0,0 +1,53 @@
1
+ $VERBOSE = true
2
+
3
+ require 'rubygems'
4
+ require "test-unit"
5
+ require 'simplecov'
6
+ SimpleCov.start
7
+
8
+ RECREATE = false
9
+
10
+ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
11
+ require "hbase-jruby"
12
+
13
+ # Required
14
+ HBase.resolve_dependency! 'cdh4.1.2'
15
+
16
+ class TestHBaseJRubyBase < Test::Unit::TestCase
17
+ TABLE = 'test_hbase_jruby'
18
+ ZK = ENV.fetch 'HBASE_JRUBY_TEST_ZK'
19
+
20
+ # Initialize
21
+ hbase = HBase.new 'hbase.zookeeper.quorum' => ZK
22
+ hbase.table(TABLE) do |table|
23
+ table.drop! if table.exists?
24
+ end
25
+ hbase.close
26
+
27
+ def setup
28
+ @hbase ||= HBase.new 'hbase.zookeeper.quorum' => ZK
29
+ @table = @hbase.table(TABLE)
30
+
31
+ # Drop & Create
32
+ @table.drop! if RECREATE && @table.exists?
33
+ @table.create!(
34
+ :cf1 => { :compression => :none, :bloomfilter => :row },
35
+ :cf2 => { :bloomfilter => :rowcol },
36
+ :cf3 => { :versions => 1, :bloomfilter => org.apache.hadoop.hbase.regionserver.StoreFile::BloomType::ROWCOL }
37
+ ) unless @table.exists?
38
+ @table.enable! if @table.disabled?
39
+
40
+ unless RECREATE
41
+ @table.each do |row|
42
+ @table.delete row.rowkey :raw
43
+ end
44
+ assert_equal 0, @table.count
45
+ end
46
+ end
47
+
48
+ def teardown
49
+ if RECREATE
50
+ @table.drop! if @table && @table.exists?
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
4
+ require 'helper'
5
+
6
+ class TestByteArray < Test::Unit::TestCase
7
+ def test_order
8
+ [
9
+ [(1..100).to_a, :fixnum],
10
+ [('aa'..'zz').to_a, :string],
11
+ ].each do |pair|
12
+ arr, type = pair
13
+ assert_equal arr,
14
+ arr.reverse.map { |e| HBase::ByteArray.new(e) }.sort.map { |ba|
15
+ HBase::Util.from_bytes type, ba.java
16
+ }
17
+ end
18
+ end
19
+
20
+ def test_stopkey_bytes_for_prefix
21
+ assert_equal HBase::ByteArray.new("hellp"),
22
+ HBase::ByteArray.new( HBase::ByteArray.new("hello").stopkey_bytes_for_prefix )
23
+ assert_equal HBase::ByteArray.new("BLUF"),
24
+ HBase::ByteArray.new( HBase::ByteArray.new("BLUE").stopkey_bytes_for_prefix )
25
+ assert_nil HBase::ByteArray.new([127, 127, 127].to_java(Java::byte)).stopkey_bytes_for_prefix
26
+ assert_equal HBase::ByteArray.new([126, 127].to_java(Java::byte)),
27
+ HBase::ByteArray.new(
28
+ HBase::ByteArray.new([126, 126, 127, 127, 127, 127].to_java(Java::byte)).stopkey_bytes_for_prefix
29
+ )
30
+ end
31
+
32
+ def test_as_hash_key
33
+ hash = {
34
+ HBase::ByteArray.new("Hello") => 1,
35
+ HBase::ByteArray.new("World") => 2
36
+ }
37
+ assert_equal 1, hash[ HBase::ByteArray.new("Hello") ]
38
+ assert_equal 2, hash[ HBase::ByteArray.new("World".to_java_bytes) ]
39
+ end
40
+ end
@@ -0,0 +1,51 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
4
+ require 'helper'
5
+ require 'bigdecimal'
6
+
7
+ class TestCell < Test::Unit::TestCase
8
+ import org.apache.hadoop.hbase.KeyValue
9
+ Util = HBase::Util
10
+
11
+ def test_cell
12
+ ts = Time.now.to_i * 1000
13
+ {
14
+ 'value' => :string,
15
+ :value => :symbol,
16
+ 123 => :fixnum,
17
+ 123456789012345678901234567890 => :bignum,
18
+ 123.456 => :float,
19
+ true => :boolean,
20
+ false => :boolean,
21
+ BigDecimal.new("123.456") => :bigdecimal
22
+ }.each do |value, type|
23
+ kv = KeyValue.new("rowkey".to_java_bytes, "hello".to_java_bytes, "world".to_java_bytes, ts, Util.to_bytes(value))
24
+ cell = HBase::Cell.new(kv)
25
+
26
+ assert_equal "rowkey", cell.rowkey
27
+ assert_equal "hello", cell.cf, cell.family
28
+ assert_equal "world", cell.cq, cell.qualifier
29
+ assert_equal ts, cell.ts
30
+ assert_equal value, cell.send(type)
31
+ assert_equal HBase::ColumnKey.new(:hello, :world), cell.column_key
32
+ assert_instance_of String, cell.inspect
33
+ end
34
+ end
35
+
36
+ def test_order
37
+ ts = Time.now.to_i * 1000
38
+
39
+ val = "val".to_java_bytes
40
+ cells =
41
+ [
42
+ KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "alpha".to_java_bytes, ts, val),
43
+ KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "alpha".to_java_bytes, ts - 1000, val),
44
+ KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "beta".to_java_bytes, ts, val),
45
+ KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "beta".to_java_bytes, ts, val),
46
+ KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "gamma".to_java_bytes, ts, val),
47
+ ].map { |kv| HBase::Cell.new kv }
48
+
49
+ assert_equal cells, cells.reverse.sort
50
+ end
51
+ end
@@ -0,0 +1,49 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
4
+ require 'helper'
5
+
6
+ class TestColumnKey < Test::Unit::TestCase
7
+ Util = HBase::Util
8
+
9
+ def test_types
10
+ ck = HBase::ColumnKey("hello", "world")
11
+ assert_equal "hello", ck.family
12
+ assert_equal "hello", ck.cf
13
+ assert_equal "world", ck.qualifier
14
+ assert_equal "world", ck.cq
15
+ assert_equal 'hello:world', ck.to_s
16
+
17
+ ck = HBase::ColumnKey("hello".to_java_bytes, 123)
18
+ assert_equal "hello", ck.family
19
+ assert_equal "hello", ck.cf
20
+ assert_equal 123, ck.qualifier(:fixnum)
21
+ assert_equal 123, ck.cq(:fixnum)
22
+
23
+ ck = HBase::ColumnKey(:hello, nil)
24
+ assert_equal "hello", ck.family
25
+ assert_equal "hello", ck.cf
26
+ assert_equal '', ck.qualifier(:string)
27
+ assert_equal '', ck.cq(:string)
28
+ assert_equal 'hello', ck.to_s
29
+ end
30
+
31
+ def test_eql
32
+ ck1 = HBase::ColumnKey(:hello, :world)
33
+ ck2 = HBase::ColumnKey("hello", "world")
34
+
35
+ assert_equal ck1, ck2
36
+ assert_equal ck1, "hello:world"
37
+ end
38
+
39
+ def test_order
40
+ assert_equal (1..100).to_a,
41
+ (1..100).to_a.reverse.map { |cq|
42
+ HBase::ColumnKey(:cf, cq)
43
+ }.sort.map { |ck| ck.cq :fixnum }
44
+ end
45
+
46
+ def test_as_hash_key
47
+ assert({ HBase::ColumnKey(:hello, :world) => true }[ ck2 = HBase::ColumnKey("hello", "world") ])
48
+ end
49
+ end
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
4
+ require 'helper'
5
+
6
+ class TestHBase < TestHBaseJRubyBase
7
+ def test_tables
8
+ assert @hbase.table_names.include?(TABLE)
9
+ assert @hbase.tables.map(&:name).include?(TABLE)
10
+ end
11
+
12
+ def test_close
13
+ @hbase.close
14
+
15
+ # TODO: Still usable after close?
16
+ assert @hbase.table_names.include?(TABLE)
17
+ assert_equal 1, @table.put('rowkey' => { 'cf1:a' => 1 })
18
+ end
19
+
20
+ def test_admin
21
+ assert_instance_of org.apache.hadoop.hbase.client.HBaseAdmin, @hbase.admin
22
+ @hbase.admin do |admin|
23
+ assert_instance_of org.apache.hadoop.hbase.client.HBaseAdmin, admin
24
+ end
25
+ end
26
+
27
+ def test_config
28
+ assert_instance_of org.apache.hadoop.conf.Configuration, @hbase.config
29
+ end
30
+
31
+ def test_shared_config
32
+ hbase2 = HBase.new @hbase.config
33
+ assert_equal @hbase.config, hbase2.config
34
+ end
35
+ end
36
+
@@ -0,0 +1,318 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path('..', __FILE__)
4
+ require 'helper'
5
+
6
+ class TestScoped < TestHBaseJRubyBase
7
+ def test_invalid_limit
8
+ assert_raise(ArgumentError) { @table.limit }
9
+ assert_raise(ArgumentError) { @table.limit(-1) }
10
+ assert_raise(ArgumentError) { @table.limit("hello") }
11
+ end
12
+
13
+ def test_invalid_versions
14
+ assert_raise(ArgumentError) { @table.versions }
15
+ assert_raise(ArgumentError) { @table.versions(0) }
16
+ assert_raise(ArgumentError) { @table.versions("hello") }
17
+ end
18
+
19
+ def test_invalid_batch
20
+ assert_raise(ArgumentError) { @table.batch }
21
+ assert_raise(ArgumentError) { @table.batch(0) }
22
+ assert_raise(ArgumentError) { @table.batch("hello") }
23
+ end
24
+
25
+ def test_invalid_range
26
+ assert_raise(ArgumentError) { @table.range }
27
+ assert_raise(ArgumentError) { @table.range(:xxx => 'row1') }
28
+ assert_raise(ArgumentError) { @table.range(1, 2, 3) }
29
+ end
30
+
31
+ def test_invalid_project
32
+ assert_raise(ArgumentError) { @table.project(:offset => 'a', :limit => 10).to_a }
33
+ assert_raise(ArgumentError) { @table.project(:offset => 10, :limit => 'a').to_a }
34
+
35
+ @table.project(:offset => 100) # Fine yet
36
+ @table.project(:limit => 10)
37
+ assert_raise(ArgumentError) { @table.project(:offset => 100).to_a }
38
+ assert_raise(ArgumentError) { @table.project(:limit => 10).to_a }
39
+ assert_raise(ArgumentError) { @table.project(:offset => -1) }
40
+ assert_raise(ArgumentError) { @table.project(:limit => -1) }
41
+ assert_raise(ArgumentError) { @table.project(:offset => :a) }
42
+ assert_raise(ArgumentError) { @table.project(:limit => :a) }
43
+ assert_raise(ArgumentError) { @table.project(:xxx => 1) }
44
+ end
45
+
46
+ def test_invalid_filter
47
+ assert_raise(ArgumentError) { @table.filter(3.14) }
48
+ assert_raise(ArgumentError) { @table.filter('cf1:a' => { xxx: 50 }) }
49
+ assert_raise(ArgumentError) { @table.filter('cf1:a' => { eq: { 1 => 2 } }) }
50
+ end
51
+
52
+ def test_each_and_count
53
+ (101..150).each do |i|
54
+ @table.put(i, 'cf1:a' => i, 'cf2:b' => i, 'cf3:c' => i * 3)
55
+ end
56
+
57
+ assert_instance_of HBase::Scoped, @table.each
58
+ scoped = @table.each
59
+ assert_equal scoped, scoped.each
60
+
61
+ assert_equal 50, @table.count
62
+ assert_equal 50, @table.each.count
63
+ assert_equal 50, @table.to_a.length # each
64
+
65
+ # Start key
66
+ assert_equal 40, @table.range(111).count
67
+
68
+ # Stop key (exclusive)
69
+ assert_equal 19, @table.range(nil, 120).count
70
+
71
+ # Start key ~ Stop key (exclusive)
72
+ assert_equal 9, @table.range(111, 120).count
73
+
74
+ # Start key ~ Stop key (exclusive)
75
+ assert_equal 9, @table.range(111...120).count
76
+
77
+ # Start key ~ Stop key (inclusive)
78
+ assert_equal 10, @table.range(111..120).count
79
+
80
+ # Start key ~ Stop key (inclusive) + limit
81
+ begin
82
+ assert_equal 5, @table.range(111..120).limit(5).count
83
+ rescue NotImplementedError
84
+ end
85
+
86
+ # Start key ~ Stop key (inclusive) + filters
87
+ assert_equal 10, @table.range(111..150).filter('cf1:a' => 131..140).count
88
+ assert_equal 9, @table.range(111..150).filter('cf1:a' => 131...140).count
89
+ assert_equal 2, @table.range(111..150).filter('cf1:a' => 131...140, 'cf2:b' => 132..133).count
90
+
91
+ # Unscope
92
+ assert_equal 50, @table.range(111..150).filter('cf1:a' => 131...140, 'cf2:b' => 132..133).unscope.count
93
+ end
94
+ def test_scan
95
+ insert = lambda do
96
+ (40..70).each do |i|
97
+ @table.put(i, 'cf1:a' => i, 'cf2:b' => i * 2, 'cf3:c' => i * 3, 'cf3:d' => 'dummy', 'cf3:e' => 3.14)
98
+ end
99
+ end
100
+ insert.call
101
+
102
+ assert_instance_of HBase::Scoped, @table.each
103
+
104
+ # Test both for HBase::Table and HBase::Scoped
105
+ [@table, @table.each].each do |table|
106
+ # project
107
+ project_cols = ['cf1:a', 'cf3:c']
108
+ assert table.project(*project_cols).all? { |result|
109
+ result.to_hash.keys == project_cols
110
+ }
111
+
112
+ # project: additive
113
+ assert_equal project_cols + ['cf3:d'], table.project(*project_cols).project('cf3:d').first.to_hash.keys.map(&:to_s)
114
+
115
+ # project: family
116
+ assert_equal %w[cf1:a cf3:c cf3:d cf3:e], table.project('cf1:a', 'cf3').first.to_hash.keys.map(&:to_s)
117
+
118
+ # filter: Hash
119
+ # to_a.length instead of count :)
120
+ assert_equal 1, table.filter('cf1:a' => 50).to_a.length
121
+ assert_equal 3, table.filter('cf1:a' => [50, 60, 70]).to_a.length
122
+ assert_equal 2, table.filter('cf1:a' => [50, 60, 70], 'cf2:b' => [100, 140]).to_a.length
123
+ assert_equal 20, table.filter('cf1:a' => [41..50, 55, 61...70]).to_a.length
124
+ assert_equal 12, table.filter('cf1:a' => [41..50, 61, 70]).to_a.length
125
+ assert_equal 0, table.filter('cf1:a' => 50, 'cf2:b' => 60).to_a.length
126
+ assert_equal 1, table.filter('cf1:a' => 50, 'cf2:b' => 90..100).to_a.length
127
+ assert_equal 0, table.filter('cf1:a' => 50, 'cf2:b' => 90...100).to_a.length
128
+ assert_equal 6, table.filter('cf1:a' => 50..60, 'cf2:b' => 100..110).to_a.length
129
+ assert_equal 10, table.filter('cf1:a' => { :> => 50, :<= => 60 }).to_a.length
130
+ assert_equal 9, table.filter('cf1:a' => { :> => 50, :<= => 60, :!= => 55 }).to_a.length
131
+ assert_equal 10, table.filter('cf1:a' => { :>= => 50, :<= => 60, :!= => 55 }).to_a.length
132
+ assert_equal 9, table.filter('cf1:a' => { :>= => 50, :< => 60, :!= => 55 }).to_a.length
133
+ assert_equal 1, table.filter('cf1:a' => { :> => 50, :<= => 60, :== => 55 }).to_a.length
134
+ assert_equal 2, table.filter('cf1:a' => { :> => 50, :<= => 60, :== => [55, 57] }).to_a.length
135
+ assert_equal 9, table.filter('cf1:a' => { gte: 50, lt: 60, ne: 55 }).to_a.length
136
+ assert_equal 7, table.filter('cf1:a' => { gte: 50, lt: 60, ne: [55, 57, 59] }).to_a.length
137
+
138
+ # filter: Hash + additive
139
+ assert_equal 6, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).to_a.length
140
+
141
+ # filter: Java filter
142
+ # Bug: https://issues.apache.org/jira/browse/HBASE-6954
143
+ import org.apache.hadoop.hbase.filter.ColumnPaginationFilter
144
+ assert_equal 3, table.filter(ColumnPaginationFilter.new(3, 1)).first.to_hash.keys.length
145
+
146
+ # filter: Java filter list
147
+ import org.apache.hadoop.hbase.filter.FilterList
148
+ import org.apache.hadoop.hbase.filter.ColumnRangeFilter
149
+ assert_equal %w[cf2:b cf3:c],
150
+ table.filter(FilterList.new [
151
+ ColumnRangeFilter.new('a'.to_java_bytes, true, 'd'.to_java_bytes, true),
152
+ ColumnPaginationFilter.new(2, 1),
153
+ ]).first.to_hash.keys.map(&:to_s)
154
+
155
+
156
+ # limit with filter
157
+ begin
158
+ assert_equal 4, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).limit(4).to_a.length
159
+ rescue NotImplementedError
160
+ end
161
+
162
+ # caching: How do we know if it's working? TODO
163
+ assert_equal 6, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).caching(10).to_a.length
164
+ end
165
+
166
+ insert.call
167
+ [@table, @table.each].each do |table|
168
+ # versions
169
+ assert table.all? { |result| result.to_hash_with_versions['cf1:a'].length == 2 }
170
+ assert table.versions(1).all? { |result| result.to_hash_with_versions['cf1:a'].length == 1 }
171
+ end
172
+ end
173
+
174
+ def test_scan_on_non_string_rowkey
175
+ (1..20).each do |rk|
176
+ @table.put rk, 'cf1:a' => rk
177
+ end
178
+ assert_equal 9, @table.range(1..9).count
179
+ assert_equal [1, 2, 3, 4, 5, 6, 7, 8, 9], @table.range(1..9).map { |row| row.rowkey :integer }
180
+ assert_equal 8, @table.range(1...9).count
181
+
182
+ @table.truncate!
183
+
184
+ (1..20).each do |rk|
185
+ @table.put rk.to_s, 'cf1:a' => rk
186
+ end
187
+ assert_equal 20, @table.range('1'..'9').count
188
+ assert_equal %w[1 10 11 12 13 14 15 16 17 18 19 2 20 3 4 5 6 7 8 9], @table.range('1'..'9').map(&:rowkey)
189
+
190
+ assert_equal 19, @table.range('1'...'9').count
191
+
192
+ @table.truncate!
193
+ data = { 'cf1:1' => 1 } # doesn't matter
194
+ (1..15).each do |i|
195
+ @table.put i, data
196
+ @table.put i.to_s, data
197
+ end
198
+
199
+ assert_equal [1, 2, 3], @table.range(1..3).map { |r| r.rowkey :integer }
200
+ assert_equal %w[1 10 11 12 13 14 15 2 3], @table.range('1'..'3').map { |r| r.rowkey :string }
201
+ end
202
+
203
+ def test_non_string_column_name
204
+ @table.put 'rowkey', Hash[ (1..20).map { |cq| [HBase::ColumnKey('cf1', cq), cq] } ]
205
+
206
+ assert((1..20).all? { |cq| @table.get('rowkey').integer(HBase::ColumnKey('cf1', cq)) == cq })
207
+
208
+ assert @table.project(['cf1', 10], ['cf1', 20]).map { |r|
209
+ [r.integer(HBase::ColumnKey('cf1', 10)), r.integer(HBase::ColumnKey.new('cf1', 20))]
210
+ }.all? { |e| e == [10, 20] }
211
+
212
+ hash = @table.get('rowkey').to_hash(
213
+ HBase::ColumnKey('cf1', 1) => :fixnum,
214
+ HBase::ColumnKey('cf1', 2) => :fixnum,
215
+ )
216
+ assert_equal 1, hash[HBase::ColumnKey(:cf1, 1)]
217
+ assert_equal 2, hash[HBase::ColumnKey(:cf1, 2)]
218
+ assert_equal 3, HBase::Util.from_bytes(:fixnum, hash[HBase::ColumnKey(:cf1, 3)])
219
+ end
220
+
221
+ def test_table_descriptor
222
+ assert_instance_of org.apache.hadoop.hbase.client.UnmodifyableHTableDescriptor, @table.descriptor
223
+
224
+ # Should be read-only
225
+ assert_raise {
226
+ @table.descriptor.setMaxFileSize 100 * 1024 ** 2
227
+ }
228
+ end
229
+
230
+ def test_null_value
231
+ 10.times do |i|
232
+ @table.put i, 'cf1:nil' => i % 2 == 0 ? nil : true
233
+ end
234
+ assert_equal 10, @table.count
235
+ assert_equal 5, @table.filter('cf1:nil' => nil).count
236
+ end
237
+
238
+ def test_scoped_get_intra_row_scan
239
+ # Preparation
240
+ all_data = {}
241
+ (1..100).each do |rk|
242
+ data = {}
243
+ (1..200).each do |cq|
244
+ data[HBase::ColumnKey(:cf1, cq)] = rk + cq
245
+ end
246
+ all_data[rk] = data
247
+ end
248
+ @table.put all_data
249
+
250
+ # One simple filter (Rowkey 10 ~ 19)
251
+ scoped1 = @table.filter(HBase::ColumnKey('cf1', 100) => 110...120)
252
+ ret = scoped1.get((1..100).to_a)
253
+ assert_equal 100, ret.count
254
+ assert_equal 10, ret.compact.count
255
+
256
+ # Two filters
257
+ scoped2 = scoped1.filter(
258
+ # Rowkey 10 ~ 19 & 9 ~ 14 = 10 ~ 14
259
+ HBase::ColumnKey('cf1', 1) => 10..15
260
+ )
261
+ ret = scoped2.get((1..100).to_a)
262
+ assert_equal 100, ret.count
263
+ assert_equal 5, ret.compact.count
264
+
265
+ # Range
266
+ assert_equal 4, scoped2.range(11).get((1..100).to_a).compact.count
267
+ assert_equal 3, scoped2.range(11..13).get((1..100).to_a).compact.count
268
+ assert_equal 2, scoped2.range(11...13).get((1..100).to_a).compact.count
269
+ assert_equal 2, scoped2.range(11, 13).get((1..100).to_a).compact.count
270
+ assert_equal 3, scoped2.range(nil, 13).get((1..100).to_a).compact.count
271
+ end
272
+
273
+ def test_prefix_filter
274
+ ('aa'..'zz').each do |rk|
275
+ @table.put rk, 'cf1:a' => 1
276
+ end
277
+
278
+ assert_equal 26, @table.range(:prefix => 'c').count
279
+ assert @table.range(:prefix => 'c').get('cc')
280
+ assert_nil @table.range(:prefix => 'c').get('dd')
281
+ assert @table.range(:prefix => ['d', 'c']).get('dd')
282
+ assert_equal 52, @table.range(:prefix => ['a', 'c']).count
283
+ assert_equal 78, @table.range(:prefix => ['d', 'a', 'c']).count
284
+ assert_equal 52, @table.range(nil, 'd', :prefix => ['d', 'a', 'c']).count
285
+ assert_equal 52, @table.range('b', :prefix => ['d', 'a', 'c']).count
286
+ assert_equal 78, @table.range('a', 'e', :prefix => ['d', 'a', 'c']).count
287
+ end
288
+
289
+ def test_advanced_projection
290
+ @table.put :rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
291
+
292
+ assert_equal 26, @table.project(:prefix => 'd').first.count
293
+ assert_equal 52, @table.project(:prefix => ['d', 'f']).first.count
294
+ assert_equal 52, @table.project(:range => 'b'...'d').first.count
295
+ assert_equal 105, @table.project(:range => ['b'...'d', 'x'..'za']).first.count
296
+ assert_equal 10, @table.project(:offset => 10, :limit => 10).first.count
297
+ assert_equal 'da', @table.project(:offset => 26 * 3, :limit => 10).first.first.cq
298
+ assert_equal 10, @table.project(:offset => 26 * 3).project(:limit => 10).first.count
299
+ assert_equal 'da', @table.project(:offset => 26 * 3).project(:limit => 10).first.first.cq
300
+ end
301
+
302
+ def test_batch
303
+ @table.put :rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
304
+
305
+ assert_equal [10, 10, 6], @table.batch(10).project(:prefix => 'd').map(&:count)
306
+
307
+ # # README example
308
+ # (1..100).each do |rk|
309
+ # @table.put rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
310
+ # end
311
+ # scoped = @table.each
312
+ # scoped.range(1..100).
313
+ # project(:prefix => 'c').
314
+ # batch(10).
315
+ # map { |row| [row.rowkey(:fixnum), row.count].map(&:to_s).join ': ' }
316
+ end
317
+ end
318
+