hbase-jruby 0.1.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +623 -0
- data/Rakefile +7 -0
- data/hbase-jruby.gemspec +23 -0
- data/lib/hbase-jruby.rb +16 -0
- data/lib/hbase-jruby/admin.rb +29 -0
- data/lib/hbase-jruby/byte_array.rb +39 -0
- data/lib/hbase-jruby/cell.rb +122 -0
- data/lib/hbase-jruby/column_key.rb +63 -0
- data/lib/hbase-jruby/dependency.rb +69 -0
- data/lib/hbase-jruby/hbase.rb +77 -0
- data/lib/hbase-jruby/pom/cdh3u5.xml +40 -0
- data/lib/hbase-jruby/pom/cdh4.1.2.xml +47 -0
- data/lib/hbase-jruby/result.rb +382 -0
- data/lib/hbase-jruby/scoped.rb +489 -0
- data/lib/hbase-jruby/table.rb +486 -0
- data/lib/hbase-jruby/util.rb +171 -0
- data/lib/hbase-jruby/version.rb +5 -0
- data/test/helper.rb +53 -0
- data/test/test_byte_array.rb +40 -0
- data/test/test_cell.rb +51 -0
- data/test/test_column_key.rb +49 -0
- data/test/test_hbase.rb +36 -0
- data/test/test_scoped.rb +318 -0
- data/test/test_table.rb +211 -0
- data/test/test_table_admin.rb +148 -0
- data/test/test_util.rb +80 -0
- metadata +116 -0
@@ -0,0 +1,171 @@
|
|
1
|
+
require 'bigdecimal'
|
2
|
+
|
3
|
+
class HBase
|
4
|
+
module Util
|
5
|
+
JAVA_BYTE_ARRAY_EMPTY = [].to_java(Java::byte)
|
6
|
+
JAVA_BYTE_ARRAY_CLASS = JAVA_BYTE_ARRAY_EMPTY.java_class
|
7
|
+
|
8
|
+
class << self
|
9
|
+
# Returns byte array representation of the Ruby object
|
10
|
+
# @param [byte[]] v
|
11
|
+
# @return [byte[]]
|
12
|
+
def to_bytes v
|
13
|
+
case v
|
14
|
+
when String
|
15
|
+
v.to_java_bytes
|
16
|
+
when Fixnum
|
17
|
+
Bytes.java_send :toBytes, [Java::long], v
|
18
|
+
when Symbol
|
19
|
+
v.to_s.to_java_bytes
|
20
|
+
when Float
|
21
|
+
Bytes.java_send :toBytes, [Java::double], v
|
22
|
+
when true, false, ByteBuffer
|
23
|
+
Bytes.to_bytes v
|
24
|
+
when nil
|
25
|
+
''.to_java_bytes
|
26
|
+
when Bignum
|
27
|
+
Bytes.java_send :toBytes, [java.math.BigDecimal], java.math.BigDecimal.new(v.to_s)
|
28
|
+
when BigDecimal
|
29
|
+
Bytes.java_send :toBytes, [java.math.BigDecimal], v.to_java
|
30
|
+
when java.math.BigDecimal
|
31
|
+
Bytes.java_send :toBytes, [java.math.BigDecimal], v
|
32
|
+
else
|
33
|
+
if v.respond_to?(:java_class) && v.java_class == JAVA_BYTE_ARRAY_CLASS
|
34
|
+
v
|
35
|
+
else
|
36
|
+
raise ArgumentError.new("Don't know how to convert #{v.class} into Java bytes")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Returns Ruby object decoded from the byte array according to the given type
|
42
|
+
# @param [Symbol, Class] type Type to convert to
|
43
|
+
# @param [byte[]] val Java byte array
|
44
|
+
# @return [Object]
|
45
|
+
def from_bytes type, val
|
46
|
+
return nil if val.nil?
|
47
|
+
|
48
|
+
case type
|
49
|
+
when :string, :str
|
50
|
+
Bytes.to_string val
|
51
|
+
when :fixnum, :int, :integer
|
52
|
+
Bytes.to_long val
|
53
|
+
when :symbol, :sym
|
54
|
+
Bytes.to_string(val).to_sym
|
55
|
+
when :bignum, :bigint, :biginteger
|
56
|
+
BigDecimal.new(Bytes.to_big_decimal(val).to_s).to_i
|
57
|
+
when :bigdecimal
|
58
|
+
BigDecimal.new(Bytes.to_big_decimal(val).to_s)
|
59
|
+
when :float, :double
|
60
|
+
Bytes.to_double val
|
61
|
+
when :boolean, :bool
|
62
|
+
Bytes.to_boolean val
|
63
|
+
when :raw
|
64
|
+
val
|
65
|
+
else
|
66
|
+
raise ArgumentError, "Invalid type: #{type}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Returns a byte array with a trailing '0' byte
|
71
|
+
# @param [byte[]] v
|
72
|
+
# @return [byte[]]
|
73
|
+
def append_0 v
|
74
|
+
baos = java.io.ByteArrayOutputStream.new
|
75
|
+
baos.write v, 0, v.length
|
76
|
+
baos.write 0
|
77
|
+
baos.toByteArray
|
78
|
+
end
|
79
|
+
|
80
|
+
# Extracts a byte array pair of column family and column qualifier from the given object
|
81
|
+
# @param [Object, Array, KeyValue] col
|
82
|
+
def parse_column_name col
|
83
|
+
case col
|
84
|
+
when ColumnKey
|
85
|
+
return col.cf.to_java_bytes, col.cq(:raw)
|
86
|
+
when KeyValue
|
87
|
+
return col.getFamily, col.getQualifier
|
88
|
+
when Array
|
89
|
+
return to_bytes(col[0]), to_bytes(col[1])
|
90
|
+
when '', nil
|
91
|
+
raise ArgumentError, "Column family not specified"
|
92
|
+
else
|
93
|
+
cf, cq = KeyValue.parseColumn(col.to_s.to_java_bytes)
|
94
|
+
cq = JAVA_BYTE_ARRAY_EMPTY if cq.nil? && col.to_s[-1] == ':'
|
95
|
+
return cf, cq
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# @return [nil]
|
100
|
+
def import_java_classes!
|
101
|
+
imp = lambda { |base, classes|
|
102
|
+
base.class_eval do
|
103
|
+
classes.each do |klass|
|
104
|
+
begin
|
105
|
+
import klass
|
106
|
+
rescue NameError => e
|
107
|
+
warn e
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
}
|
112
|
+
|
113
|
+
@@imported ||= begin
|
114
|
+
imp.call HBase, %w[
|
115
|
+
org.apache.hadoop.hbase.HBaseConfiguration
|
116
|
+
org.apache.hadoop.hbase.client.HBaseAdmin
|
117
|
+
org.apache.hadoop.hbase.client.HTablePool
|
118
|
+
org.apache.hadoop.hbase.client.HConnectionManager
|
119
|
+
]
|
120
|
+
|
121
|
+
imp.call HBase::Util, %w[
|
122
|
+
java.nio.ByteBuffer
|
123
|
+
org.apache.hadoop.hbase.util.Bytes
|
124
|
+
org.apache.hadoop.hbase.KeyValue
|
125
|
+
]
|
126
|
+
|
127
|
+
imp.call HBase::ByteArray, %w[
|
128
|
+
java.util.Arrays
|
129
|
+
org.apache.hadoop.hbase.util.Bytes
|
130
|
+
]
|
131
|
+
|
132
|
+
imp.call HBase::Cell, %w[
|
133
|
+
org.apache.hadoop.hbase.KeyValue
|
134
|
+
]
|
135
|
+
|
136
|
+
imp.call HBase::ColumnKey, %w[
|
137
|
+
java.util.Arrays
|
138
|
+
org.apache.hadoop.hbase.util.Bytes
|
139
|
+
]
|
140
|
+
|
141
|
+
imp.call HBase::Table, %w[
|
142
|
+
org.apache.hadoop.hbase.HColumnDescriptor
|
143
|
+
org.apache.hadoop.hbase.HTableDescriptor
|
144
|
+
org.apache.hadoop.hbase.client.Delete
|
145
|
+
org.apache.hadoop.hbase.client.Increment
|
146
|
+
org.apache.hadoop.hbase.client.Put
|
147
|
+
org.apache.hadoop.hbase.io.hfile.Compression
|
148
|
+
org.apache.hadoop.hbase.regionserver.StoreFile
|
149
|
+
]
|
150
|
+
|
151
|
+
imp.call HBase::Scoped, %w[
|
152
|
+
org.apache.hadoop.hbase.client.Get
|
153
|
+
org.apache.hadoop.hbase.client.Scan
|
154
|
+
org.apache.hadoop.hbase.filter.BinaryComparator
|
155
|
+
org.apache.hadoop.hbase.filter.ColumnPaginationFilter
|
156
|
+
org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter
|
157
|
+
org.apache.hadoop.hbase.filter.ColumnRangeFilter
|
158
|
+
org.apache.hadoop.hbase.filter.CompareFilter
|
159
|
+
org.apache.hadoop.hbase.filter.FilterBase
|
160
|
+
org.apache.hadoop.hbase.filter.FilterList
|
161
|
+
org.apache.hadoop.hbase.filter.KeyOnlyFilter
|
162
|
+
org.apache.hadoop.hbase.filter.PrefixFilter
|
163
|
+
org.apache.hadoop.hbase.filter.RowFilter
|
164
|
+
org.apache.hadoop.hbase.filter.SingleColumnValueFilter
|
165
|
+
]
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end#Util
|
170
|
+
end#HBase
|
171
|
+
|
data/test/helper.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
$VERBOSE = true
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require "test-unit"
|
5
|
+
require 'simplecov'
|
6
|
+
SimpleCov.start
|
7
|
+
|
8
|
+
RECREATE = false
|
9
|
+
|
10
|
+
$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
11
|
+
require "hbase-jruby"
|
12
|
+
|
13
|
+
# Required
|
14
|
+
HBase.resolve_dependency! 'cdh4.1.2'
|
15
|
+
|
16
|
+
class TestHBaseJRubyBase < Test::Unit::TestCase
|
17
|
+
TABLE = 'test_hbase_jruby'
|
18
|
+
ZK = ENV.fetch 'HBASE_JRUBY_TEST_ZK'
|
19
|
+
|
20
|
+
# Initialize
|
21
|
+
hbase = HBase.new 'hbase.zookeeper.quorum' => ZK
|
22
|
+
hbase.table(TABLE) do |table|
|
23
|
+
table.drop! if table.exists?
|
24
|
+
end
|
25
|
+
hbase.close
|
26
|
+
|
27
|
+
def setup
|
28
|
+
@hbase ||= HBase.new 'hbase.zookeeper.quorum' => ZK
|
29
|
+
@table = @hbase.table(TABLE)
|
30
|
+
|
31
|
+
# Drop & Create
|
32
|
+
@table.drop! if RECREATE && @table.exists?
|
33
|
+
@table.create!(
|
34
|
+
:cf1 => { :compression => :none, :bloomfilter => :row },
|
35
|
+
:cf2 => { :bloomfilter => :rowcol },
|
36
|
+
:cf3 => { :versions => 1, :bloomfilter => org.apache.hadoop.hbase.regionserver.StoreFile::BloomType::ROWCOL }
|
37
|
+
) unless @table.exists?
|
38
|
+
@table.enable! if @table.disabled?
|
39
|
+
|
40
|
+
unless RECREATE
|
41
|
+
@table.each do |row|
|
42
|
+
@table.delete row.rowkey :raw
|
43
|
+
end
|
44
|
+
assert_equal 0, @table.count
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def teardown
|
49
|
+
if RECREATE
|
50
|
+
@table.drop! if @table && @table.exists?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('..', __FILE__)
|
4
|
+
require 'helper'
|
5
|
+
|
6
|
+
class TestByteArray < Test::Unit::TestCase
|
7
|
+
def test_order
|
8
|
+
[
|
9
|
+
[(1..100).to_a, :fixnum],
|
10
|
+
[('aa'..'zz').to_a, :string],
|
11
|
+
].each do |pair|
|
12
|
+
arr, type = pair
|
13
|
+
assert_equal arr,
|
14
|
+
arr.reverse.map { |e| HBase::ByteArray.new(e) }.sort.map { |ba|
|
15
|
+
HBase::Util.from_bytes type, ba.java
|
16
|
+
}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_stopkey_bytes_for_prefix
|
21
|
+
assert_equal HBase::ByteArray.new("hellp"),
|
22
|
+
HBase::ByteArray.new( HBase::ByteArray.new("hello").stopkey_bytes_for_prefix )
|
23
|
+
assert_equal HBase::ByteArray.new("BLUF"),
|
24
|
+
HBase::ByteArray.new( HBase::ByteArray.new("BLUE").stopkey_bytes_for_prefix )
|
25
|
+
assert_nil HBase::ByteArray.new([127, 127, 127].to_java(Java::byte)).stopkey_bytes_for_prefix
|
26
|
+
assert_equal HBase::ByteArray.new([126, 127].to_java(Java::byte)),
|
27
|
+
HBase::ByteArray.new(
|
28
|
+
HBase::ByteArray.new([126, 126, 127, 127, 127, 127].to_java(Java::byte)).stopkey_bytes_for_prefix
|
29
|
+
)
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_as_hash_key
|
33
|
+
hash = {
|
34
|
+
HBase::ByteArray.new("Hello") => 1,
|
35
|
+
HBase::ByteArray.new("World") => 2
|
36
|
+
}
|
37
|
+
assert_equal 1, hash[ HBase::ByteArray.new("Hello") ]
|
38
|
+
assert_equal 2, hash[ HBase::ByteArray.new("World".to_java_bytes) ]
|
39
|
+
end
|
40
|
+
end
|
data/test/test_cell.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('..', __FILE__)
|
4
|
+
require 'helper'
|
5
|
+
require 'bigdecimal'
|
6
|
+
|
7
|
+
class TestCell < Test::Unit::TestCase
|
8
|
+
import org.apache.hadoop.hbase.KeyValue
|
9
|
+
Util = HBase::Util
|
10
|
+
|
11
|
+
def test_cell
|
12
|
+
ts = Time.now.to_i * 1000
|
13
|
+
{
|
14
|
+
'value' => :string,
|
15
|
+
:value => :symbol,
|
16
|
+
123 => :fixnum,
|
17
|
+
123456789012345678901234567890 => :bignum,
|
18
|
+
123.456 => :float,
|
19
|
+
true => :boolean,
|
20
|
+
false => :boolean,
|
21
|
+
BigDecimal.new("123.456") => :bigdecimal
|
22
|
+
}.each do |value, type|
|
23
|
+
kv = KeyValue.new("rowkey".to_java_bytes, "hello".to_java_bytes, "world".to_java_bytes, ts, Util.to_bytes(value))
|
24
|
+
cell = HBase::Cell.new(kv)
|
25
|
+
|
26
|
+
assert_equal "rowkey", cell.rowkey
|
27
|
+
assert_equal "hello", cell.cf, cell.family
|
28
|
+
assert_equal "world", cell.cq, cell.qualifier
|
29
|
+
assert_equal ts, cell.ts
|
30
|
+
assert_equal value, cell.send(type)
|
31
|
+
assert_equal HBase::ColumnKey.new(:hello, :world), cell.column_key
|
32
|
+
assert_instance_of String, cell.inspect
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_order
|
37
|
+
ts = Time.now.to_i * 1000
|
38
|
+
|
39
|
+
val = "val".to_java_bytes
|
40
|
+
cells =
|
41
|
+
[
|
42
|
+
KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "alpha".to_java_bytes, ts, val),
|
43
|
+
KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "alpha".to_java_bytes, ts - 1000, val),
|
44
|
+
KeyValue.new("rowkey".to_java_bytes, "apple".to_java_bytes, "beta".to_java_bytes, ts, val),
|
45
|
+
KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "beta".to_java_bytes, ts, val),
|
46
|
+
KeyValue.new("rowkey".to_java_bytes, "banana".to_java_bytes, "gamma".to_java_bytes, ts, val),
|
47
|
+
].map { |kv| HBase::Cell.new kv }
|
48
|
+
|
49
|
+
assert_equal cells, cells.reverse.sort
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('..', __FILE__)
|
4
|
+
require 'helper'
|
5
|
+
|
6
|
+
class TestColumnKey < Test::Unit::TestCase
|
7
|
+
Util = HBase::Util
|
8
|
+
|
9
|
+
def test_types
|
10
|
+
ck = HBase::ColumnKey("hello", "world")
|
11
|
+
assert_equal "hello", ck.family
|
12
|
+
assert_equal "hello", ck.cf
|
13
|
+
assert_equal "world", ck.qualifier
|
14
|
+
assert_equal "world", ck.cq
|
15
|
+
assert_equal 'hello:world', ck.to_s
|
16
|
+
|
17
|
+
ck = HBase::ColumnKey("hello".to_java_bytes, 123)
|
18
|
+
assert_equal "hello", ck.family
|
19
|
+
assert_equal "hello", ck.cf
|
20
|
+
assert_equal 123, ck.qualifier(:fixnum)
|
21
|
+
assert_equal 123, ck.cq(:fixnum)
|
22
|
+
|
23
|
+
ck = HBase::ColumnKey(:hello, nil)
|
24
|
+
assert_equal "hello", ck.family
|
25
|
+
assert_equal "hello", ck.cf
|
26
|
+
assert_equal '', ck.qualifier(:string)
|
27
|
+
assert_equal '', ck.cq(:string)
|
28
|
+
assert_equal 'hello', ck.to_s
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_eql
|
32
|
+
ck1 = HBase::ColumnKey(:hello, :world)
|
33
|
+
ck2 = HBase::ColumnKey("hello", "world")
|
34
|
+
|
35
|
+
assert_equal ck1, ck2
|
36
|
+
assert_equal ck1, "hello:world"
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_order
|
40
|
+
assert_equal (1..100).to_a,
|
41
|
+
(1..100).to_a.reverse.map { |cq|
|
42
|
+
HBase::ColumnKey(:cf, cq)
|
43
|
+
}.sort.map { |ck| ck.cq :fixnum }
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_as_hash_key
|
47
|
+
assert({ HBase::ColumnKey(:hello, :world) => true }[ ck2 = HBase::ColumnKey("hello", "world") ])
|
48
|
+
end
|
49
|
+
end
|
data/test/test_hbase.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('..', __FILE__)
|
4
|
+
require 'helper'
|
5
|
+
|
6
|
+
class TestHBase < TestHBaseJRubyBase
|
7
|
+
def test_tables
|
8
|
+
assert @hbase.table_names.include?(TABLE)
|
9
|
+
assert @hbase.tables.map(&:name).include?(TABLE)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_close
|
13
|
+
@hbase.close
|
14
|
+
|
15
|
+
# TODO: Still usable after close?
|
16
|
+
assert @hbase.table_names.include?(TABLE)
|
17
|
+
assert_equal 1, @table.put('rowkey' => { 'cf1:a' => 1 })
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_admin
|
21
|
+
assert_instance_of org.apache.hadoop.hbase.client.HBaseAdmin, @hbase.admin
|
22
|
+
@hbase.admin do |admin|
|
23
|
+
assert_instance_of org.apache.hadoop.hbase.client.HBaseAdmin, admin
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_config
|
28
|
+
assert_instance_of org.apache.hadoop.conf.Configuration, @hbase.config
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_shared_config
|
32
|
+
hbase2 = HBase.new @hbase.config
|
33
|
+
assert_equal @hbase.config, hbase2.config
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
data/test/test_scoped.rb
ADDED
@@ -0,0 +1,318 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.expand_path('..', __FILE__)
|
4
|
+
require 'helper'
|
5
|
+
|
6
|
+
class TestScoped < TestHBaseJRubyBase
|
7
|
+
def test_invalid_limit
|
8
|
+
assert_raise(ArgumentError) { @table.limit }
|
9
|
+
assert_raise(ArgumentError) { @table.limit(-1) }
|
10
|
+
assert_raise(ArgumentError) { @table.limit("hello") }
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_invalid_versions
|
14
|
+
assert_raise(ArgumentError) { @table.versions }
|
15
|
+
assert_raise(ArgumentError) { @table.versions(0) }
|
16
|
+
assert_raise(ArgumentError) { @table.versions("hello") }
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_invalid_batch
|
20
|
+
assert_raise(ArgumentError) { @table.batch }
|
21
|
+
assert_raise(ArgumentError) { @table.batch(0) }
|
22
|
+
assert_raise(ArgumentError) { @table.batch("hello") }
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_invalid_range
|
26
|
+
assert_raise(ArgumentError) { @table.range }
|
27
|
+
assert_raise(ArgumentError) { @table.range(:xxx => 'row1') }
|
28
|
+
assert_raise(ArgumentError) { @table.range(1, 2, 3) }
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_invalid_project
|
32
|
+
assert_raise(ArgumentError) { @table.project(:offset => 'a', :limit => 10).to_a }
|
33
|
+
assert_raise(ArgumentError) { @table.project(:offset => 10, :limit => 'a').to_a }
|
34
|
+
|
35
|
+
@table.project(:offset => 100) # Fine yet
|
36
|
+
@table.project(:limit => 10)
|
37
|
+
assert_raise(ArgumentError) { @table.project(:offset => 100).to_a }
|
38
|
+
assert_raise(ArgumentError) { @table.project(:limit => 10).to_a }
|
39
|
+
assert_raise(ArgumentError) { @table.project(:offset => -1) }
|
40
|
+
assert_raise(ArgumentError) { @table.project(:limit => -1) }
|
41
|
+
assert_raise(ArgumentError) { @table.project(:offset => :a) }
|
42
|
+
assert_raise(ArgumentError) { @table.project(:limit => :a) }
|
43
|
+
assert_raise(ArgumentError) { @table.project(:xxx => 1) }
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_invalid_filter
|
47
|
+
assert_raise(ArgumentError) { @table.filter(3.14) }
|
48
|
+
assert_raise(ArgumentError) { @table.filter('cf1:a' => { xxx: 50 }) }
|
49
|
+
assert_raise(ArgumentError) { @table.filter('cf1:a' => { eq: { 1 => 2 } }) }
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_each_and_count
|
53
|
+
(101..150).each do |i|
|
54
|
+
@table.put(i, 'cf1:a' => i, 'cf2:b' => i, 'cf3:c' => i * 3)
|
55
|
+
end
|
56
|
+
|
57
|
+
assert_instance_of HBase::Scoped, @table.each
|
58
|
+
scoped = @table.each
|
59
|
+
assert_equal scoped, scoped.each
|
60
|
+
|
61
|
+
assert_equal 50, @table.count
|
62
|
+
assert_equal 50, @table.each.count
|
63
|
+
assert_equal 50, @table.to_a.length # each
|
64
|
+
|
65
|
+
# Start key
|
66
|
+
assert_equal 40, @table.range(111).count
|
67
|
+
|
68
|
+
# Stop key (exclusive)
|
69
|
+
assert_equal 19, @table.range(nil, 120).count
|
70
|
+
|
71
|
+
# Start key ~ Stop key (exclusive)
|
72
|
+
assert_equal 9, @table.range(111, 120).count
|
73
|
+
|
74
|
+
# Start key ~ Stop key (exclusive)
|
75
|
+
assert_equal 9, @table.range(111...120).count
|
76
|
+
|
77
|
+
# Start key ~ Stop key (inclusive)
|
78
|
+
assert_equal 10, @table.range(111..120).count
|
79
|
+
|
80
|
+
# Start key ~ Stop key (inclusive) + limit
|
81
|
+
begin
|
82
|
+
assert_equal 5, @table.range(111..120).limit(5).count
|
83
|
+
rescue NotImplementedError
|
84
|
+
end
|
85
|
+
|
86
|
+
# Start key ~ Stop key (inclusive) + filters
|
87
|
+
assert_equal 10, @table.range(111..150).filter('cf1:a' => 131..140).count
|
88
|
+
assert_equal 9, @table.range(111..150).filter('cf1:a' => 131...140).count
|
89
|
+
assert_equal 2, @table.range(111..150).filter('cf1:a' => 131...140, 'cf2:b' => 132..133).count
|
90
|
+
|
91
|
+
# Unscope
|
92
|
+
assert_equal 50, @table.range(111..150).filter('cf1:a' => 131...140, 'cf2:b' => 132..133).unscope.count
|
93
|
+
end
|
94
|
+
def test_scan
|
95
|
+
insert = lambda do
|
96
|
+
(40..70).each do |i|
|
97
|
+
@table.put(i, 'cf1:a' => i, 'cf2:b' => i * 2, 'cf3:c' => i * 3, 'cf3:d' => 'dummy', 'cf3:e' => 3.14)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
insert.call
|
101
|
+
|
102
|
+
assert_instance_of HBase::Scoped, @table.each
|
103
|
+
|
104
|
+
# Test both for HBase::Table and HBase::Scoped
|
105
|
+
[@table, @table.each].each do |table|
|
106
|
+
# project
|
107
|
+
project_cols = ['cf1:a', 'cf3:c']
|
108
|
+
assert table.project(*project_cols).all? { |result|
|
109
|
+
result.to_hash.keys == project_cols
|
110
|
+
}
|
111
|
+
|
112
|
+
# project: additive
|
113
|
+
assert_equal project_cols + ['cf3:d'], table.project(*project_cols).project('cf3:d').first.to_hash.keys.map(&:to_s)
|
114
|
+
|
115
|
+
# project: family
|
116
|
+
assert_equal %w[cf1:a cf3:c cf3:d cf3:e], table.project('cf1:a', 'cf3').first.to_hash.keys.map(&:to_s)
|
117
|
+
|
118
|
+
# filter: Hash
|
119
|
+
# to_a.length instead of count :)
|
120
|
+
assert_equal 1, table.filter('cf1:a' => 50).to_a.length
|
121
|
+
assert_equal 3, table.filter('cf1:a' => [50, 60, 70]).to_a.length
|
122
|
+
assert_equal 2, table.filter('cf1:a' => [50, 60, 70], 'cf2:b' => [100, 140]).to_a.length
|
123
|
+
assert_equal 20, table.filter('cf1:a' => [41..50, 55, 61...70]).to_a.length
|
124
|
+
assert_equal 12, table.filter('cf1:a' => [41..50, 61, 70]).to_a.length
|
125
|
+
assert_equal 0, table.filter('cf1:a' => 50, 'cf2:b' => 60).to_a.length
|
126
|
+
assert_equal 1, table.filter('cf1:a' => 50, 'cf2:b' => 90..100).to_a.length
|
127
|
+
assert_equal 0, table.filter('cf1:a' => 50, 'cf2:b' => 90...100).to_a.length
|
128
|
+
assert_equal 6, table.filter('cf1:a' => 50..60, 'cf2:b' => 100..110).to_a.length
|
129
|
+
assert_equal 10, table.filter('cf1:a' => { :> => 50, :<= => 60 }).to_a.length
|
130
|
+
assert_equal 9, table.filter('cf1:a' => { :> => 50, :<= => 60, :!= => 55 }).to_a.length
|
131
|
+
assert_equal 10, table.filter('cf1:a' => { :>= => 50, :<= => 60, :!= => 55 }).to_a.length
|
132
|
+
assert_equal 9, table.filter('cf1:a' => { :>= => 50, :< => 60, :!= => 55 }).to_a.length
|
133
|
+
assert_equal 1, table.filter('cf1:a' => { :> => 50, :<= => 60, :== => 55 }).to_a.length
|
134
|
+
assert_equal 2, table.filter('cf1:a' => { :> => 50, :<= => 60, :== => [55, 57] }).to_a.length
|
135
|
+
assert_equal 9, table.filter('cf1:a' => { gte: 50, lt: 60, ne: 55 }).to_a.length
|
136
|
+
assert_equal 7, table.filter('cf1:a' => { gte: 50, lt: 60, ne: [55, 57, 59] }).to_a.length
|
137
|
+
|
138
|
+
# filter: Hash + additive
|
139
|
+
assert_equal 6, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).to_a.length
|
140
|
+
|
141
|
+
# filter: Java filter
|
142
|
+
# Bug: https://issues.apache.org/jira/browse/HBASE-6954
|
143
|
+
import org.apache.hadoop.hbase.filter.ColumnPaginationFilter
|
144
|
+
assert_equal 3, table.filter(ColumnPaginationFilter.new(3, 1)).first.to_hash.keys.length
|
145
|
+
|
146
|
+
# filter: Java filter list
|
147
|
+
import org.apache.hadoop.hbase.filter.FilterList
|
148
|
+
import org.apache.hadoop.hbase.filter.ColumnRangeFilter
|
149
|
+
assert_equal %w[cf2:b cf3:c],
|
150
|
+
table.filter(FilterList.new [
|
151
|
+
ColumnRangeFilter.new('a'.to_java_bytes, true, 'd'.to_java_bytes, true),
|
152
|
+
ColumnPaginationFilter.new(2, 1),
|
153
|
+
]).first.to_hash.keys.map(&:to_s)
|
154
|
+
|
155
|
+
|
156
|
+
# limit with filter
|
157
|
+
begin
|
158
|
+
assert_equal 4, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).limit(4).to_a.length
|
159
|
+
rescue NotImplementedError
|
160
|
+
end
|
161
|
+
|
162
|
+
# caching: How do we know if it's working? TODO
|
163
|
+
assert_equal 6, table.filter('cf1:a' => 50..60).filter('cf2:b' => 100..110).caching(10).to_a.length
|
164
|
+
end
|
165
|
+
|
166
|
+
insert.call
|
167
|
+
[@table, @table.each].each do |table|
|
168
|
+
# versions
|
169
|
+
assert table.all? { |result| result.to_hash_with_versions['cf1:a'].length == 2 }
|
170
|
+
assert table.versions(1).all? { |result| result.to_hash_with_versions['cf1:a'].length == 1 }
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
def test_scan_on_non_string_rowkey
|
175
|
+
(1..20).each do |rk|
|
176
|
+
@table.put rk, 'cf1:a' => rk
|
177
|
+
end
|
178
|
+
assert_equal 9, @table.range(1..9).count
|
179
|
+
assert_equal [1, 2, 3, 4, 5, 6, 7, 8, 9], @table.range(1..9).map { |row| row.rowkey :integer }
|
180
|
+
assert_equal 8, @table.range(1...9).count
|
181
|
+
|
182
|
+
@table.truncate!
|
183
|
+
|
184
|
+
(1..20).each do |rk|
|
185
|
+
@table.put rk.to_s, 'cf1:a' => rk
|
186
|
+
end
|
187
|
+
assert_equal 20, @table.range('1'..'9').count
|
188
|
+
assert_equal %w[1 10 11 12 13 14 15 16 17 18 19 2 20 3 4 5 6 7 8 9], @table.range('1'..'9').map(&:rowkey)
|
189
|
+
|
190
|
+
assert_equal 19, @table.range('1'...'9').count
|
191
|
+
|
192
|
+
@table.truncate!
|
193
|
+
data = { 'cf1:1' => 1 } # doesn't matter
|
194
|
+
(1..15).each do |i|
|
195
|
+
@table.put i, data
|
196
|
+
@table.put i.to_s, data
|
197
|
+
end
|
198
|
+
|
199
|
+
assert_equal [1, 2, 3], @table.range(1..3).map { |r| r.rowkey :integer }
|
200
|
+
assert_equal %w[1 10 11 12 13 14 15 2 3], @table.range('1'..'3').map { |r| r.rowkey :string }
|
201
|
+
end
|
202
|
+
|
203
|
+
def test_non_string_column_name
|
204
|
+
@table.put 'rowkey', Hash[ (1..20).map { |cq| [HBase::ColumnKey('cf1', cq), cq] } ]
|
205
|
+
|
206
|
+
assert((1..20).all? { |cq| @table.get('rowkey').integer(HBase::ColumnKey('cf1', cq)) == cq })
|
207
|
+
|
208
|
+
assert @table.project(['cf1', 10], ['cf1', 20]).map { |r|
|
209
|
+
[r.integer(HBase::ColumnKey('cf1', 10)), r.integer(HBase::ColumnKey.new('cf1', 20))]
|
210
|
+
}.all? { |e| e == [10, 20] }
|
211
|
+
|
212
|
+
hash = @table.get('rowkey').to_hash(
|
213
|
+
HBase::ColumnKey('cf1', 1) => :fixnum,
|
214
|
+
HBase::ColumnKey('cf1', 2) => :fixnum,
|
215
|
+
)
|
216
|
+
assert_equal 1, hash[HBase::ColumnKey(:cf1, 1)]
|
217
|
+
assert_equal 2, hash[HBase::ColumnKey(:cf1, 2)]
|
218
|
+
assert_equal 3, HBase::Util.from_bytes(:fixnum, hash[HBase::ColumnKey(:cf1, 3)])
|
219
|
+
end
|
220
|
+
|
221
|
+
def test_table_descriptor
|
222
|
+
assert_instance_of org.apache.hadoop.hbase.client.UnmodifyableHTableDescriptor, @table.descriptor
|
223
|
+
|
224
|
+
# Should be read-only
|
225
|
+
assert_raise {
|
226
|
+
@table.descriptor.setMaxFileSize 100 * 1024 ** 2
|
227
|
+
}
|
228
|
+
end
|
229
|
+
|
230
|
+
def test_null_value
|
231
|
+
10.times do |i|
|
232
|
+
@table.put i, 'cf1:nil' => i % 2 == 0 ? nil : true
|
233
|
+
end
|
234
|
+
assert_equal 10, @table.count
|
235
|
+
assert_equal 5, @table.filter('cf1:nil' => nil).count
|
236
|
+
end
|
237
|
+
|
238
|
+
def test_scoped_get_intra_row_scan
|
239
|
+
# Preparation
|
240
|
+
all_data = {}
|
241
|
+
(1..100).each do |rk|
|
242
|
+
data = {}
|
243
|
+
(1..200).each do |cq|
|
244
|
+
data[HBase::ColumnKey(:cf1, cq)] = rk + cq
|
245
|
+
end
|
246
|
+
all_data[rk] = data
|
247
|
+
end
|
248
|
+
@table.put all_data
|
249
|
+
|
250
|
+
# One simple filter (Rowkey 10 ~ 19)
|
251
|
+
scoped1 = @table.filter(HBase::ColumnKey('cf1', 100) => 110...120)
|
252
|
+
ret = scoped1.get((1..100).to_a)
|
253
|
+
assert_equal 100, ret.count
|
254
|
+
assert_equal 10, ret.compact.count
|
255
|
+
|
256
|
+
# Two filters
|
257
|
+
scoped2 = scoped1.filter(
|
258
|
+
# Rowkey 10 ~ 19 & 9 ~ 14 = 10 ~ 14
|
259
|
+
HBase::ColumnKey('cf1', 1) => 10..15
|
260
|
+
)
|
261
|
+
ret = scoped2.get((1..100).to_a)
|
262
|
+
assert_equal 100, ret.count
|
263
|
+
assert_equal 5, ret.compact.count
|
264
|
+
|
265
|
+
# Range
|
266
|
+
assert_equal 4, scoped2.range(11).get((1..100).to_a).compact.count
|
267
|
+
assert_equal 3, scoped2.range(11..13).get((1..100).to_a).compact.count
|
268
|
+
assert_equal 2, scoped2.range(11...13).get((1..100).to_a).compact.count
|
269
|
+
assert_equal 2, scoped2.range(11, 13).get((1..100).to_a).compact.count
|
270
|
+
assert_equal 3, scoped2.range(nil, 13).get((1..100).to_a).compact.count
|
271
|
+
end
|
272
|
+
|
273
|
+
def test_prefix_filter
|
274
|
+
('aa'..'zz').each do |rk|
|
275
|
+
@table.put rk, 'cf1:a' => 1
|
276
|
+
end
|
277
|
+
|
278
|
+
assert_equal 26, @table.range(:prefix => 'c').count
|
279
|
+
assert @table.range(:prefix => 'c').get('cc')
|
280
|
+
assert_nil @table.range(:prefix => 'c').get('dd')
|
281
|
+
assert @table.range(:prefix => ['d', 'c']).get('dd')
|
282
|
+
assert_equal 52, @table.range(:prefix => ['a', 'c']).count
|
283
|
+
assert_equal 78, @table.range(:prefix => ['d', 'a', 'c']).count
|
284
|
+
assert_equal 52, @table.range(nil, 'd', :prefix => ['d', 'a', 'c']).count
|
285
|
+
assert_equal 52, @table.range('b', :prefix => ['d', 'a', 'c']).count
|
286
|
+
assert_equal 78, @table.range('a', 'e', :prefix => ['d', 'a', 'c']).count
|
287
|
+
end
|
288
|
+
|
289
|
+
def test_advanced_projection
|
290
|
+
@table.put :rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
|
291
|
+
|
292
|
+
assert_equal 26, @table.project(:prefix => 'd').first.count
|
293
|
+
assert_equal 52, @table.project(:prefix => ['d', 'f']).first.count
|
294
|
+
assert_equal 52, @table.project(:range => 'b'...'d').first.count
|
295
|
+
assert_equal 105, @table.project(:range => ['b'...'d', 'x'..'za']).first.count
|
296
|
+
assert_equal 10, @table.project(:offset => 10, :limit => 10).first.count
|
297
|
+
assert_equal 'da', @table.project(:offset => 26 * 3, :limit => 10).first.first.cq
|
298
|
+
assert_equal 10, @table.project(:offset => 26 * 3).project(:limit => 10).first.count
|
299
|
+
assert_equal 'da', @table.project(:offset => 26 * 3).project(:limit => 10).first.first.cq
|
300
|
+
end
|
301
|
+
|
302
|
+
def test_batch
|
303
|
+
@table.put :rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
|
304
|
+
|
305
|
+
assert_equal [10, 10, 6], @table.batch(10).project(:prefix => 'd').map(&:count)
|
306
|
+
|
307
|
+
# # README example
|
308
|
+
# (1..100).each do |rk|
|
309
|
+
# @table.put rk, Hash[ ('aa'..'zz').map { |cq| [ "cf1:#{cq}", 100 ] } ]
|
310
|
+
# end
|
311
|
+
# scoped = @table.each
|
312
|
+
# scoped.range(1..100).
|
313
|
+
# project(:prefix => 'c').
|
314
|
+
# batch(10).
|
315
|
+
# map { |row| [row.rowkey(:fixnum), row.count].map(&:to_s).join ': ' }
|
316
|
+
end
|
317
|
+
end
|
318
|
+
|